blob: 17d67b579beb0c3f6f97bfd8add5d7027741854a [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * NET3 Protocol independent device support routines.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *
9 * Derived from the non IP parts of dev.c 1.0.19
Jesper Juhl02c30a82005-05-05 16:16:16 -070010 * Authors: Ross Biro
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Mark Evans, <evansmp@uhura.aston.ac.uk>
13 *
14 * Additional Authors:
15 * Florian la Roche <rzsfl@rz.uni-sb.de>
16 * Alan Cox <gw4pts@gw4pts.ampr.org>
17 * David Hinds <dahinds@users.sourceforge.net>
18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19 * Adam Sulmicki <adam@cfar.umd.edu>
20 * Pekka Riikonen <priikone@poesidon.pspt.fi>
21 *
22 * Changes:
23 * D.J. Barrow : Fixed bug where dev->refcnt gets set
24 * to 2 if register_netdev gets called
25 * before net_dev_init & also removed a
26 * few lines of code in the process.
27 * Alan Cox : device private ioctl copies fields back.
28 * Alan Cox : Transmit queue code does relevant
29 * stunts to keep the queue safe.
30 * Alan Cox : Fixed double lock.
31 * Alan Cox : Fixed promisc NULL pointer trap
32 * ???????? : Support the full private ioctl range
33 * Alan Cox : Moved ioctl permission check into
34 * drivers
35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
36 * Alan Cox : 100 backlog just doesn't cut it when
37 * you start doing multicast video 8)
38 * Alan Cox : Rewrote net_bh and list manager.
39 * Alan Cox : Fix ETH_P_ALL echoback lengths.
40 * Alan Cox : Took out transmit every packet pass
41 * Saved a few bytes in the ioctl handler
42 * Alan Cox : Network driver sets packet type before
43 * calling netif_rx. Saves a function
44 * call a packet.
45 * Alan Cox : Hashed net_bh()
46 * Richard Kooijman: Timestamp fixes.
47 * Alan Cox : Wrong field in SIOCGIFDSTADDR
48 * Alan Cox : Device lock protection.
49 * Alan Cox : Fixed nasty side effect of device close
50 * changes.
51 * Rudi Cilibrasi : Pass the right thing to
52 * set_mac_address()
53 * Dave Miller : 32bit quantity for the device lock to
54 * make it work out on a Sparc.
55 * Bjorn Ekwall : Added KERNELD hack.
56 * Alan Cox : Cleaned up the backlog initialise.
57 * Craig Metz : SIOCGIFCONF fix if space for under
58 * 1 device.
59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
60 * is no device open function.
61 * Andi Kleen : Fix error reporting for SIOCGIFCONF
62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
63 * Cyrus Durgin : Cleaned for KMOD
64 * Adam Sulmicki : Bug Fix : Network Device Unload
65 * A network device unload needs to purge
66 * the backlog queue.
67 * Paul Rusty Russell : SIOCSIFNAME
68 * Pekka Riikonen : Netdev boot-time settings code
69 * Andrew Morton : Make unregister_netdevice wait
70 * indefinitely on dev->refcnt
71 * J Hadi Salim : - Backlog queue sampling
72 * - netif_rx() feedback
73 */
74
75#include <asm/uaccess.h>
76#include <asm/system.h>
77#include <linux/bitops.h>
Randy Dunlap4fc268d2006-01-11 12:17:47 -080078#include <linux/capability.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070079#include <linux/cpu.h>
80#include <linux/types.h>
81#include <linux/kernel.h>
stephen hemminger08e98972009-11-10 07:20:34 +000082#include <linux/hash.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090083#include <linux/slab.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070084#include <linux/sched.h>
Arjan van de Ven4a3e2f72006-03-20 22:33:17 -080085#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070086#include <linux/string.h>
87#include <linux/mm.h>
88#include <linux/socket.h>
89#include <linux/sockios.h>
90#include <linux/errno.h>
91#include <linux/interrupt.h>
92#include <linux/if_ether.h>
93#include <linux/netdevice.h>
94#include <linux/etherdevice.h>
Ben Hutchings0187bdf2008-06-19 16:15:47 -070095#include <linux/ethtool.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070096#include <linux/notifier.h>
97#include <linux/skbuff.h>
Eric W. Biederman457c4cb2007-09-12 12:01:34 +020098#include <net/net_namespace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070099#include <net/sock.h>
100#include <linux/rtnetlink.h>
101#include <linux/proc_fs.h>
102#include <linux/seq_file.h>
103#include <linux/stat.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700104#include <net/dst.h>
105#include <net/pkt_sched.h>
106#include <net/checksum.h>
Arnd Bergmann44540962009-11-26 06:07:08 +0000107#include <net/xfrm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700108#include <linux/highmem.h>
109#include <linux/init.h>
110#include <linux/kmod.h>
111#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112#include <linux/netpoll.h>
113#include <linux/rcupdate.h>
114#include <linux/delay.h>
Johannes Berg295f4a12007-04-26 20:43:56 -0700115#include <net/wext.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700116#include <net/iw_handler.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700117#include <asm/current.h>
Steve Grubb5bdb9882005-12-03 08:39:35 -0500118#include <linux/audit.h>
Chris Leechdb217332006-06-17 21:24:58 -0700119#include <linux/dmaengine.h>
Herbert Xuf6a78bf2006-06-22 02:57:17 -0700120#include <linux/err.h>
David S. Millerc7fa9d12006-08-15 16:34:13 -0700121#include <linux/ctype.h>
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700122#include <linux/if_arp.h>
Ben Hutchings6de329e2008-06-16 17:02:28 -0700123#include <linux/if_vlan.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700124#include <linux/ip.h>
Alexander Duyckad55dca2008-09-20 22:05:50 -0700125#include <net/ip.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700126#include <linux/ipv6.h>
127#include <linux/in.h>
David S. Millerb6b2fed2008-07-21 09:48:06 -0700128#include <linux/jhash.h>
129#include <linux/random.h>
David S. Miller9cbc1cb2009-06-15 03:02:23 -0700130#include <trace/events/napi.h>
Koki Sanagicf66ba52010-08-23 18:45:02 +0900131#include <trace/events/net.h>
Koki Sanagi07dc22e2010-08-23 18:46:12 +0900132#include <trace/events/skb.h>
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +0000133#include <linux/pci.h>
Stephen Rothwellcaeda9b2010-09-16 21:39:16 -0700134#include <linux/inetdevice.h>
Ben Hutchingsc4454772011-01-19 11:03:53 +0000135#include <linux/cpu_rmap.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700136
Pavel Emelyanov342709e2007-10-23 21:14:45 -0700137#include "net-sysfs.h"
138
Herbert Xud565b0a2008-12-15 23:38:52 -0800139/* Instead of increasing this, you should create a hash table. */
140#define MAX_GRO_SKBS 8
141
Herbert Xu5d38a072009-01-04 16:13:40 -0800142/* This should be increased if a protocol with a bigger head is added. */
143#define GRO_MAX_HEAD (MAX_HEADER + 128)
144
Linus Torvalds1da177e2005-04-16 15:20:36 -0700145/*
146 * The list of packet types we will receive (as opposed to discard)
147 * and the routines to invoke.
148 *
149 * Why 16. Because with 16 the only overlap we get on a hash of the
150 * low nibble of the protocol value is RARP/SNAP/X.25.
151 *
152 * NOTE: That is no longer true with the addition of VLAN tags. Not
153 * sure which should go first, but I bet it won't make much
154 * difference if we are running VLANs. The good news is that
155 * this protocol won't be in the list unless compiled in, so
Stephen Hemminger3041a062006-05-26 13:25:24 -0700156 * the average user (w/out VLANs) will not be adversely affected.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700157 * --BLG
158 *
159 * 0800 IP
160 * 8100 802.1Q VLAN
161 * 0001 802.3
162 * 0002 AX.25
163 * 0004 802.2
164 * 8035 RARP
165 * 0005 SNAP
166 * 0805 X.25
167 * 0806 ARP
168 * 8137 IPX
169 * 0009 Localtalk
170 * 86DD IPv6
171 */
172
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +0800173#define PTYPE_HASH_SIZE (16)
174#define PTYPE_HASH_MASK (PTYPE_HASH_SIZE - 1)
175
Linus Torvalds1da177e2005-04-16 15:20:36 -0700176static DEFINE_SPINLOCK(ptype_lock);
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +0800177static struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -0700178static struct list_head ptype_all __read_mostly; /* Taps */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700179
Linus Torvalds1da177e2005-04-16 15:20:36 -0700180/*
Pavel Emelianov7562f872007-05-03 15:13:45 -0700181 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
Linus Torvalds1da177e2005-04-16 15:20:36 -0700182 * semaphore.
183 *
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800184 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
Linus Torvalds1da177e2005-04-16 15:20:36 -0700185 *
186 * Writers must hold the rtnl semaphore while they loop through the
Pavel Emelianov7562f872007-05-03 15:13:45 -0700187 * dev_base_head list, and hold dev_base_lock for writing when they do the
Linus Torvalds1da177e2005-04-16 15:20:36 -0700188 * actual updates. This allows pure readers to access the list even
189 * while a writer is preparing to update it.
190 *
191 * To put it another way, dev_base_lock is held for writing only to
192 * protect against pure readers; the rtnl semaphore provides the
193 * protection against other writers.
194 *
195 * See, for example usages, register_netdevice() and
196 * unregister_netdevice(), which must be called with the rtnl
197 * semaphore held.
198 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700199DEFINE_RWLOCK(dev_base_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700200EXPORT_SYMBOL(dev_base_lock);
201
Thomas Graf4e985ad2011-06-21 03:11:20 +0000202static inline void dev_base_seq_inc(struct net *net)
203{
204 while (++net->dev_base_seq == 0);
205}
206
Eric W. Biederman881d9662007-09-17 11:56:21 -0700207static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700208{
209 unsigned hash = full_name_hash(name, strnlen(name, IFNAMSIZ));
stephen hemminger08e98972009-11-10 07:20:34 +0000210 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700211}
212
Eric W. Biederman881d9662007-09-17 11:56:21 -0700213static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700214{
Eric Dumazet7c28bd02009-10-24 06:13:17 -0700215 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700216}
217
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000218static inline void rps_lock(struct softnet_data *sd)
Changli Gao152102c2010-03-30 20:16:22 +0000219{
220#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000221 spin_lock(&sd->input_pkt_queue.lock);
Changli Gao152102c2010-03-30 20:16:22 +0000222#endif
223}
224
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000225static inline void rps_unlock(struct softnet_data *sd)
Changli Gao152102c2010-03-30 20:16:22 +0000226{
227#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000228 spin_unlock(&sd->input_pkt_queue.lock);
Changli Gao152102c2010-03-30 20:16:22 +0000229#endif
230}
231
Eric W. Biedermance286d32007-09-12 13:53:49 +0200232/* Device list insertion */
233static int list_netdevice(struct net_device *dev)
234{
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900235 struct net *net = dev_net(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200236
237 ASSERT_RTNL();
238
239 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800240 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
Eric Dumazet72c95282009-10-30 07:11:27 +0000241 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000242 hlist_add_head_rcu(&dev->index_hlist,
243 dev_index_hash(net, dev->ifindex));
Eric W. Biedermance286d32007-09-12 13:53:49 +0200244 write_unlock_bh(&dev_base_lock);
Thomas Graf4e985ad2011-06-21 03:11:20 +0000245
246 dev_base_seq_inc(net);
247
Eric W. Biedermance286d32007-09-12 13:53:49 +0200248 return 0;
249}
250
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000251/* Device list removal
252 * caller must respect a RCU grace period before freeing/reusing dev
253 */
Eric W. Biedermance286d32007-09-12 13:53:49 +0200254static void unlist_netdevice(struct net_device *dev)
255{
256 ASSERT_RTNL();
257
258 /* Unlink dev from the device chain */
259 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800260 list_del_rcu(&dev->dev_list);
Eric Dumazet72c95282009-10-30 07:11:27 +0000261 hlist_del_rcu(&dev->name_hlist);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000262 hlist_del_rcu(&dev->index_hlist);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200263 write_unlock_bh(&dev_base_lock);
Thomas Graf4e985ad2011-06-21 03:11:20 +0000264
265 dev_base_seq_inc(dev_net(dev));
Eric W. Biedermance286d32007-09-12 13:53:49 +0200266}
267
Linus Torvalds1da177e2005-04-16 15:20:36 -0700268/*
269 * Our notifier list
270 */
271
Alan Sternf07d5b92006-05-09 15:23:03 -0700272static RAW_NOTIFIER_HEAD(netdev_chain);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700273
274/*
275 * Device drivers call our routines to queue packets here. We empty the
276 * queue in the local softnet handler.
277 */
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700278
Eric Dumazet9958da02010-04-17 04:17:02 +0000279DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700280EXPORT_PER_CPU_SYMBOL(softnet_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700281
David S. Millercf508b12008-07-22 14:16:42 -0700282#ifdef CONFIG_LOCKDEP
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700283/*
David S. Millerc773e842008-07-08 23:13:53 -0700284 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700285 * according to dev->type
286 */
287static const unsigned short netdev_lock_type[] =
288 {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
289 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
290 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
291 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
292 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
293 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
294 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
295 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
296 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
297 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
298 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
299 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
300 ARPHRD_FCFABRIC, ARPHRD_IEEE802_TR, ARPHRD_IEEE80211,
Rémi Denis-Courmont2d91d782008-12-17 15:47:29 -0800301 ARPHRD_IEEE80211_PRISM, ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET,
Dmitry Eremin-Solenikov929122cd2009-08-14 20:00:20 +0400302 ARPHRD_PHONET_PIPE, ARPHRD_IEEE802154,
Sergey Lapinfcb94e42009-06-08 12:18:47 +0000303 ARPHRD_VOID, ARPHRD_NONE};
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700304
Jan Engelhardt36cbd3d2009-08-05 10:42:58 -0700305static const char *const netdev_lock_name[] =
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700306 {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
307 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
308 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
309 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
310 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
311 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
312 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
313 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
314 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
315 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
316 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
317 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
318 "_xmit_FCFABRIC", "_xmit_IEEE802_TR", "_xmit_IEEE80211",
Rémi Denis-Courmont2d91d782008-12-17 15:47:29 -0800319 "_xmit_IEEE80211_PRISM", "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET",
Dmitry Eremin-Solenikov929122cd2009-08-14 20:00:20 +0400320 "_xmit_PHONET_PIPE", "_xmit_IEEE802154",
Sergey Lapinfcb94e42009-06-08 12:18:47 +0000321 "_xmit_VOID", "_xmit_NONE"};
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700322
323static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
David S. Millercf508b12008-07-22 14:16:42 -0700324static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700325
326static inline unsigned short netdev_lock_pos(unsigned short dev_type)
327{
328 int i;
329
330 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
331 if (netdev_lock_type[i] == dev_type)
332 return i;
333 /* the last key is used by default */
334 return ARRAY_SIZE(netdev_lock_type) - 1;
335}
336
David S. Millercf508b12008-07-22 14:16:42 -0700337static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
338 unsigned short dev_type)
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700339{
340 int i;
341
342 i = netdev_lock_pos(dev_type);
343 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
344 netdev_lock_name[i]);
345}
David S. Millercf508b12008-07-22 14:16:42 -0700346
347static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
348{
349 int i;
350
351 i = netdev_lock_pos(dev->type);
352 lockdep_set_class_and_name(&dev->addr_list_lock,
353 &netdev_addr_lock_key[i],
354 netdev_lock_name[i]);
355}
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700356#else
David S. Millercf508b12008-07-22 14:16:42 -0700357static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
358 unsigned short dev_type)
359{
360}
361static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700362{
363}
364#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700365
366/*******************************************************************************
367
368 Protocol management and registration routines
369
370*******************************************************************************/
371
372/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700373 * Add a protocol ID to the list. Now that the input handler is
374 * smarter we can dispense with all the messy stuff that used to be
375 * here.
376 *
377 * BEWARE!!! Protocol handlers, mangling input packets,
378 * MUST BE last in hash buckets and checking protocol handlers
379 * MUST start from promiscuous ptype_all chain in net_bh.
380 * It is true now, do not change it.
381 * Explanation follows: if protocol handler, mangling packet, will
382 * be the first on list, it is not able to sense, that packet
383 * is cloned and should be copied-on-write, so that it will
384 * change it and subsequent readers will get broken packet.
385 * --ANK (980803)
386 */
387
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000388static inline struct list_head *ptype_head(const struct packet_type *pt)
389{
390 if (pt->type == htons(ETH_P_ALL))
391 return &ptype_all;
392 else
393 return &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
394}
395
Linus Torvalds1da177e2005-04-16 15:20:36 -0700396/**
397 * dev_add_pack - add packet handler
398 * @pt: packet type declaration
399 *
400 * Add a protocol handler to the networking stack. The passed &packet_type
401 * is linked into kernel lists and may not be freed until it has been
402 * removed from the kernel lists.
403 *
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900404 * This call does not sleep therefore it can not
Linus Torvalds1da177e2005-04-16 15:20:36 -0700405 * guarantee all CPU's that are in middle of receiving packets
406 * will see the new packet type (until the next received packet).
407 */
408
409void dev_add_pack(struct packet_type *pt)
410{
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000411 struct list_head *head = ptype_head(pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700412
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000413 spin_lock(&ptype_lock);
414 list_add_rcu(&pt->list, head);
415 spin_unlock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700416}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700417EXPORT_SYMBOL(dev_add_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700418
Linus Torvalds1da177e2005-04-16 15:20:36 -0700419/**
420 * __dev_remove_pack - remove packet handler
421 * @pt: packet type declaration
422 *
423 * Remove a protocol handler that was previously added to the kernel
424 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
425 * from the kernel lists and can be freed or reused once this function
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900426 * returns.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700427 *
428 * The packet type might still be in use by receivers
429 * and must not be freed until after all the CPU's have gone
430 * through a quiescent state.
431 */
432void __dev_remove_pack(struct packet_type *pt)
433{
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000434 struct list_head *head = ptype_head(pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700435 struct packet_type *pt1;
436
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000437 spin_lock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700438
439 list_for_each_entry(pt1, head, list) {
440 if (pt == pt1) {
441 list_del_rcu(&pt->list);
442 goto out;
443 }
444 }
445
446 printk(KERN_WARNING "dev_remove_pack: %p not found.\n", pt);
447out:
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000448 spin_unlock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700449}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700450EXPORT_SYMBOL(__dev_remove_pack);
451
Linus Torvalds1da177e2005-04-16 15:20:36 -0700452/**
453 * dev_remove_pack - remove packet handler
454 * @pt: packet type declaration
455 *
456 * Remove a protocol handler that was previously added to the kernel
457 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
458 * from the kernel lists and can be freed or reused once this function
459 * returns.
460 *
461 * This call sleeps to guarantee that no CPU is looking at the packet
462 * type after return.
463 */
464void dev_remove_pack(struct packet_type *pt)
465{
466 __dev_remove_pack(pt);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900467
Linus Torvalds1da177e2005-04-16 15:20:36 -0700468 synchronize_net();
469}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700470EXPORT_SYMBOL(dev_remove_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700471
472/******************************************************************************
473
474 Device Boot-time Settings Routines
475
476*******************************************************************************/
477
478/* Boot time configuration table */
479static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
480
481/**
482 * netdev_boot_setup_add - add new setup entry
483 * @name: name of the device
484 * @map: configured settings for the device
485 *
486 * Adds new setup entry to the dev_boot_setup list. The function
487 * returns 0 on error and 1 on success. This is a generic routine to
488 * all netdevices.
489 */
490static int netdev_boot_setup_add(char *name, struct ifmap *map)
491{
492 struct netdev_boot_setup *s;
493 int i;
494
495 s = dev_boot_setup;
496 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
497 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
498 memset(s[i].name, 0, sizeof(s[i].name));
Wang Chen93b3cff2008-07-01 19:57:19 -0700499 strlcpy(s[i].name, name, IFNAMSIZ);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700500 memcpy(&s[i].map, map, sizeof(s[i].map));
501 break;
502 }
503 }
504
505 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
506}
507
508/**
509 * netdev_boot_setup_check - check boot time settings
510 * @dev: the netdevice
511 *
512 * Check boot time settings for the device.
513 * The found settings are set for the device to be used
514 * later in the device probing.
515 * Returns 0 if no settings found, 1 if they are.
516 */
517int netdev_boot_setup_check(struct net_device *dev)
518{
519 struct netdev_boot_setup *s = dev_boot_setup;
520 int i;
521
522 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
523 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
Wang Chen93b3cff2008-07-01 19:57:19 -0700524 !strcmp(dev->name, s[i].name)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700525 dev->irq = s[i].map.irq;
526 dev->base_addr = s[i].map.base_addr;
527 dev->mem_start = s[i].map.mem_start;
528 dev->mem_end = s[i].map.mem_end;
529 return 1;
530 }
531 }
532 return 0;
533}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700534EXPORT_SYMBOL(netdev_boot_setup_check);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700535
536
537/**
538 * netdev_boot_base - get address from boot time settings
539 * @prefix: prefix for network device
540 * @unit: id for network device
541 *
542 * Check boot time settings for the base address of device.
543 * The found settings are set for the device to be used
544 * later in the device probing.
545 * Returns 0 if no settings found.
546 */
547unsigned long netdev_boot_base(const char *prefix, int unit)
548{
549 const struct netdev_boot_setup *s = dev_boot_setup;
550 char name[IFNAMSIZ];
551 int i;
552
553 sprintf(name, "%s%d", prefix, unit);
554
555 /*
556 * If device already registered then return base of 1
557 * to indicate not to probe for this interface
558 */
Eric W. Biederman881d9662007-09-17 11:56:21 -0700559 if (__dev_get_by_name(&init_net, name))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700560 return 1;
561
562 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
563 if (!strcmp(name, s[i].name))
564 return s[i].map.base_addr;
565 return 0;
566}
567
568/*
569 * Saves at boot time configured settings for any netdevice.
570 */
571int __init netdev_boot_setup(char *str)
572{
573 int ints[5];
574 struct ifmap map;
575
576 str = get_options(str, ARRAY_SIZE(ints), ints);
577 if (!str || !*str)
578 return 0;
579
580 /* Save settings */
581 memset(&map, 0, sizeof(map));
582 if (ints[0] > 0)
583 map.irq = ints[1];
584 if (ints[0] > 1)
585 map.base_addr = ints[2];
586 if (ints[0] > 2)
587 map.mem_start = ints[3];
588 if (ints[0] > 3)
589 map.mem_end = ints[4];
590
591 /* Add new entry to the list */
592 return netdev_boot_setup_add(str, &map);
593}
594
595__setup("netdev=", netdev_boot_setup);
596
597/*******************************************************************************
598
599 Device Interface Subroutines
600
601*******************************************************************************/
602
603/**
604 * __dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700605 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700606 * @name: name to find
607 *
608 * Find an interface by name. Must be called under RTNL semaphore
609 * or @dev_base_lock. If the name is found a pointer to the device
610 * is returned. If the name is not found then %NULL is returned. The
611 * reference counters are not incremented so the caller must be
612 * careful with locks.
613 */
614
Eric W. Biederman881d9662007-09-17 11:56:21 -0700615struct net_device *__dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700616{
617 struct hlist_node *p;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700618 struct net_device *dev;
619 struct hlist_head *head = dev_name_hash(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700620
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700621 hlist_for_each_entry(dev, p, head, name_hlist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700622 if (!strncmp(dev->name, name, IFNAMSIZ))
623 return dev;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700624
Linus Torvalds1da177e2005-04-16 15:20:36 -0700625 return NULL;
626}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700627EXPORT_SYMBOL(__dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700628
629/**
Eric Dumazet72c95282009-10-30 07:11:27 +0000630 * dev_get_by_name_rcu - find a device by its name
631 * @net: the applicable net namespace
632 * @name: name to find
633 *
634 * Find an interface by name.
635 * If the name is found a pointer to the device is returned.
636 * If the name is not found then %NULL is returned.
637 * The reference counters are not incremented so the caller must be
638 * careful with locks. The caller must hold RCU lock.
639 */
640
641struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
642{
643 struct hlist_node *p;
644 struct net_device *dev;
645 struct hlist_head *head = dev_name_hash(net, name);
646
647 hlist_for_each_entry_rcu(dev, p, head, name_hlist)
648 if (!strncmp(dev->name, name, IFNAMSIZ))
649 return dev;
650
651 return NULL;
652}
653EXPORT_SYMBOL(dev_get_by_name_rcu);
654
655/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700656 * dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700657 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700658 * @name: name to find
659 *
660 * Find an interface by name. This can be called from any
661 * context and does its own locking. The returned handle has
662 * the usage count incremented and the caller must use dev_put() to
663 * release it when it is no longer needed. %NULL is returned if no
664 * matching device is found.
665 */
666
Eric W. Biederman881d9662007-09-17 11:56:21 -0700667struct net_device *dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700668{
669 struct net_device *dev;
670
Eric Dumazet72c95282009-10-30 07:11:27 +0000671 rcu_read_lock();
672 dev = dev_get_by_name_rcu(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700673 if (dev)
674 dev_hold(dev);
Eric Dumazet72c95282009-10-30 07:11:27 +0000675 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700676 return dev;
677}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700678EXPORT_SYMBOL(dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700679
680/**
681 * __dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700682 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700683 * @ifindex: index of device
684 *
685 * Search for an interface by index. Returns %NULL if the device
686 * is not found or a pointer to the device. The device has not
687 * had its reference counter increased so the caller must be careful
688 * about locking. The caller must hold either the RTNL semaphore
689 * or @dev_base_lock.
690 */
691
Eric W. Biederman881d9662007-09-17 11:56:21 -0700692struct net_device *__dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700693{
694 struct hlist_node *p;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700695 struct net_device *dev;
696 struct hlist_head *head = dev_index_hash(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700697
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700698 hlist_for_each_entry(dev, p, head, index_hlist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700699 if (dev->ifindex == ifindex)
700 return dev;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700701
Linus Torvalds1da177e2005-04-16 15:20:36 -0700702 return NULL;
703}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700704EXPORT_SYMBOL(__dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700705
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000706/**
707 * dev_get_by_index_rcu - find a device by its ifindex
708 * @net: the applicable net namespace
709 * @ifindex: index of device
710 *
711 * Search for an interface by index. Returns %NULL if the device
712 * is not found or a pointer to the device. The device has not
713 * had its reference counter increased so the caller must be careful
714 * about locking. The caller must hold RCU lock.
715 */
716
717struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
718{
719 struct hlist_node *p;
720 struct net_device *dev;
721 struct hlist_head *head = dev_index_hash(net, ifindex);
722
723 hlist_for_each_entry_rcu(dev, p, head, index_hlist)
724 if (dev->ifindex == ifindex)
725 return dev;
726
727 return NULL;
728}
729EXPORT_SYMBOL(dev_get_by_index_rcu);
730
Linus Torvalds1da177e2005-04-16 15:20:36 -0700731
732/**
733 * dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700734 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700735 * @ifindex: index of device
736 *
737 * Search for an interface by index. Returns NULL if the device
738 * is not found or a pointer to the device. The device returned has
739 * had a reference added and the pointer is safe until the user calls
740 * dev_put to indicate they have finished with it.
741 */
742
Eric W. Biederman881d9662007-09-17 11:56:21 -0700743struct net_device *dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700744{
745 struct net_device *dev;
746
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000747 rcu_read_lock();
748 dev = dev_get_by_index_rcu(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700749 if (dev)
750 dev_hold(dev);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000751 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700752 return dev;
753}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700754EXPORT_SYMBOL(dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700755
756/**
Eric Dumazet941666c2010-12-05 01:23:53 +0000757 * dev_getbyhwaddr_rcu - find a device by its hardware address
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700758 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700759 * @type: media type of device
760 * @ha: hardware address
761 *
762 * Search for an interface by MAC address. Returns NULL if the device
Eric Dumazetc5066532011-01-24 13:16:16 -0800763 * is not found or a pointer to the device.
764 * The caller must hold RCU or RTNL.
Eric Dumazet941666c2010-12-05 01:23:53 +0000765 * The returned device has not had its ref count increased
Linus Torvalds1da177e2005-04-16 15:20:36 -0700766 * and the caller must therefore be careful about locking
767 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700768 */
769
Eric Dumazet941666c2010-12-05 01:23:53 +0000770struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
771 const char *ha)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700772{
773 struct net_device *dev;
774
Eric Dumazet941666c2010-12-05 01:23:53 +0000775 for_each_netdev_rcu(net, dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700776 if (dev->type == type &&
777 !memcmp(dev->dev_addr, ha, dev->addr_len))
Pavel Emelianov7562f872007-05-03 15:13:45 -0700778 return dev;
779
780 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700781}
Eric Dumazet941666c2010-12-05 01:23:53 +0000782EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
Jochen Friedrichcf309e32005-09-22 04:44:55 -0300783
Eric W. Biederman881d9662007-09-17 11:56:21 -0700784struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700785{
786 struct net_device *dev;
787
788 ASSERT_RTNL();
Eric W. Biederman881d9662007-09-17 11:56:21 -0700789 for_each_netdev(net, dev)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700790 if (dev->type == type)
Pavel Emelianov7562f872007-05-03 15:13:45 -0700791 return dev;
792
793 return NULL;
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700794}
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700795EXPORT_SYMBOL(__dev_getfirstbyhwtype);
796
Eric W. Biederman881d9662007-09-17 11:56:21 -0700797struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700798{
Eric Dumazet99fe3c32010-03-18 11:27:25 +0000799 struct net_device *dev, *ret = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700800
Eric Dumazet99fe3c32010-03-18 11:27:25 +0000801 rcu_read_lock();
802 for_each_netdev_rcu(net, dev)
803 if (dev->type == type) {
804 dev_hold(dev);
805 ret = dev;
806 break;
807 }
808 rcu_read_unlock();
809 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700810}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700811EXPORT_SYMBOL(dev_getfirstbyhwtype);
812
813/**
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000814 * dev_get_by_flags_rcu - find any device with given flags
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700815 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700816 * @if_flags: IFF_* values
817 * @mask: bitmask of bits in if_flags to check
818 *
819 * Search for any interface with the given flags. Returns NULL if a device
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000820 * is not found or a pointer to the device. Must be called inside
821 * rcu_read_lock(), and result refcount is unchanged.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700822 */
823
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000824struct net_device *dev_get_by_flags_rcu(struct net *net, unsigned short if_flags,
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700825 unsigned short mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700826{
Pavel Emelianov7562f872007-05-03 15:13:45 -0700827 struct net_device *dev, *ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700828
Pavel Emelianov7562f872007-05-03 15:13:45 -0700829 ret = NULL;
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800830 for_each_netdev_rcu(net, dev) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700831 if (((dev->flags ^ if_flags) & mask) == 0) {
Pavel Emelianov7562f872007-05-03 15:13:45 -0700832 ret = dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700833 break;
834 }
835 }
Pavel Emelianov7562f872007-05-03 15:13:45 -0700836 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700837}
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000838EXPORT_SYMBOL(dev_get_by_flags_rcu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700839
840/**
841 * dev_valid_name - check if name is okay for network device
842 * @name: name string
843 *
844 * Network device names need to be valid file names to
David S. Millerc7fa9d12006-08-15 16:34:13 -0700845 * to allow sysfs to work. We also disallow any kind of
846 * whitespace.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700847 */
Mitch Williamsc2373ee2005-11-09 10:34:45 -0800848int dev_valid_name(const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700849{
David S. Millerc7fa9d12006-08-15 16:34:13 -0700850 if (*name == '\0')
851 return 0;
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -0700852 if (strlen(name) >= IFNAMSIZ)
853 return 0;
David S. Millerc7fa9d12006-08-15 16:34:13 -0700854 if (!strcmp(name, ".") || !strcmp(name, ".."))
855 return 0;
856
857 while (*name) {
858 if (*name == '/' || isspace(*name))
859 return 0;
860 name++;
861 }
862 return 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700863}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700864EXPORT_SYMBOL(dev_valid_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700865
866/**
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200867 * __dev_alloc_name - allocate a name for a device
868 * @net: network namespace to allocate the device name in
Linus Torvalds1da177e2005-04-16 15:20:36 -0700869 * @name: name format string
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200870 * @buf: scratch buffer and result name string
Linus Torvalds1da177e2005-04-16 15:20:36 -0700871 *
872 * Passed a format string - eg "lt%d" it will try and find a suitable
Stephen Hemminger3041a062006-05-26 13:25:24 -0700873 * id. It scans list of devices to build up a free map, then chooses
874 * the first empty slot. The caller must hold the dev_base or rtnl lock
875 * while allocating the name and adding the device in order to avoid
876 * duplicates.
877 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
878 * Returns the number of the unit assigned or a negative errno code.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700879 */
880
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200881static int __dev_alloc_name(struct net *net, const char *name, char *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700882{
883 int i = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700884 const char *p;
885 const int max_netdevices = 8*PAGE_SIZE;
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -0700886 unsigned long *inuse;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700887 struct net_device *d;
888
889 p = strnchr(name, IFNAMSIZ-1, '%');
890 if (p) {
891 /*
892 * Verify the string as this thing may have come from
893 * the user. There must be either one "%d" and no other "%"
894 * characters.
895 */
896 if (p[1] != 'd' || strchr(p + 2, '%'))
897 return -EINVAL;
898
899 /* Use one page as a bit array of possible slots */
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -0700900 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700901 if (!inuse)
902 return -ENOMEM;
903
Eric W. Biederman881d9662007-09-17 11:56:21 -0700904 for_each_netdev(net, d) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700905 if (!sscanf(d->name, name, &i))
906 continue;
907 if (i < 0 || i >= max_netdevices)
908 continue;
909
910 /* avoid cases where sscanf is not exact inverse of printf */
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200911 snprintf(buf, IFNAMSIZ, name, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700912 if (!strncmp(buf, d->name, IFNAMSIZ))
913 set_bit(i, inuse);
914 }
915
916 i = find_first_zero_bit(inuse, max_netdevices);
917 free_page((unsigned long) inuse);
918 }
919
Octavian Purdilad9031022009-11-18 02:36:59 +0000920 if (buf != name)
921 snprintf(buf, IFNAMSIZ, name, i);
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200922 if (!__dev_get_by_name(net, buf))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700923 return i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700924
925 /* It is possible to run out of possible slots
926 * when the name is long and there isn't enough space left
927 * for the digits, or if all bits are used.
928 */
929 return -ENFILE;
930}
931
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200932/**
933 * dev_alloc_name - allocate a name for a device
934 * @dev: device
935 * @name: name format string
936 *
937 * Passed a format string - eg "lt%d" it will try and find a suitable
938 * id. It scans list of devices to build up a free map, then chooses
939 * the first empty slot. The caller must hold the dev_base or rtnl lock
940 * while allocating the name and adding the device in order to avoid
941 * duplicates.
942 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
943 * Returns the number of the unit assigned or a negative errno code.
944 */
945
946int dev_alloc_name(struct net_device *dev, const char *name)
947{
948 char buf[IFNAMSIZ];
949 struct net *net;
950 int ret;
951
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900952 BUG_ON(!dev_net(dev));
953 net = dev_net(dev);
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200954 ret = __dev_alloc_name(net, name, buf);
955 if (ret >= 0)
956 strlcpy(dev->name, buf, IFNAMSIZ);
957 return ret;
958}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700959EXPORT_SYMBOL(dev_alloc_name);
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200960
Jiri Pirko1c5cae82011-04-30 01:21:32 +0000961static int dev_get_valid_name(struct net_device *dev, const char *name)
Octavian Purdilad9031022009-11-18 02:36:59 +0000962{
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +0000963 struct net *net;
964
965 BUG_ON(!dev_net(dev));
966 net = dev_net(dev);
967
Octavian Purdilad9031022009-11-18 02:36:59 +0000968 if (!dev_valid_name(name))
969 return -EINVAL;
970
Jiri Pirko1c5cae82011-04-30 01:21:32 +0000971 if (strchr(name, '%'))
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +0000972 return dev_alloc_name(dev, name);
Octavian Purdilad9031022009-11-18 02:36:59 +0000973 else if (__dev_get_by_name(net, name))
974 return -EEXIST;
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +0000975 else if (dev->name != name)
976 strlcpy(dev->name, name, IFNAMSIZ);
Octavian Purdilad9031022009-11-18 02:36:59 +0000977
978 return 0;
979}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700980
981/**
982 * dev_change_name - change name of a device
983 * @dev: device
984 * @newname: name (or format string) must be at least IFNAMSIZ
985 *
986 * Change name of a device, can pass format strings "eth%d".
987 * for wildcarding.
988 */
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -0700989int dev_change_name(struct net_device *dev, const char *newname)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700990{
Herbert Xufcc5a032007-07-30 17:03:38 -0700991 char oldname[IFNAMSIZ];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700992 int err = 0;
Herbert Xufcc5a032007-07-30 17:03:38 -0700993 int ret;
Eric W. Biederman881d9662007-09-17 11:56:21 -0700994 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700995
996 ASSERT_RTNL();
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900997 BUG_ON(!dev_net(dev));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700998
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900999 net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001000 if (dev->flags & IFF_UP)
1001 return -EBUSY;
1002
Stephen Hemmingerc8d90dc2007-10-26 03:53:42 -07001003 if (strncmp(newname, dev->name, IFNAMSIZ) == 0)
1004 return 0;
1005
Herbert Xufcc5a032007-07-30 17:03:38 -07001006 memcpy(oldname, dev->name, IFNAMSIZ);
1007
Jiri Pirko1c5cae82011-04-30 01:21:32 +00001008 err = dev_get_valid_name(dev, newname);
Octavian Purdilad9031022009-11-18 02:36:59 +00001009 if (err < 0)
1010 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001011
Herbert Xufcc5a032007-07-30 17:03:38 -07001012rollback:
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07001013 ret = device_rename(&dev->dev, dev->name);
1014 if (ret) {
1015 memcpy(dev->name, oldname, IFNAMSIZ);
1016 return ret;
Stephen Hemmingerdcc99772008-05-14 22:33:38 -07001017 }
Herbert Xu7f988ea2007-07-30 16:35:46 -07001018
1019 write_lock_bh(&dev_base_lock);
Eric Dumazet372b2312011-05-17 13:56:59 -04001020 hlist_del_rcu(&dev->name_hlist);
Eric Dumazet72c95282009-10-30 07:11:27 +00001021 write_unlock_bh(&dev_base_lock);
1022
1023 synchronize_rcu();
1024
1025 write_lock_bh(&dev_base_lock);
1026 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
Herbert Xu7f988ea2007-07-30 16:35:46 -07001027 write_unlock_bh(&dev_base_lock);
1028
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001029 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001030 ret = notifier_to_errno(ret);
1031
1032 if (ret) {
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001033 /* err >= 0 after dev_alloc_name() or stores the first errno */
1034 if (err >= 0) {
Herbert Xufcc5a032007-07-30 17:03:38 -07001035 err = ret;
1036 memcpy(dev->name, oldname, IFNAMSIZ);
1037 goto rollback;
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001038 } else {
1039 printk(KERN_ERR
1040 "%s: name change rollback failed: %d.\n",
1041 dev->name, ret);
Herbert Xufcc5a032007-07-30 17:03:38 -07001042 }
1043 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001044
1045 return err;
1046}
1047
1048/**
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001049 * dev_set_alias - change ifalias of a device
1050 * @dev: device
1051 * @alias: name up to IFALIASZ
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07001052 * @len: limit of bytes to copy from info
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001053 *
1054 * Set ifalias for a device,
1055 */
1056int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1057{
1058 ASSERT_RTNL();
1059
1060 if (len >= IFALIASZ)
1061 return -EINVAL;
1062
Oliver Hartkopp96ca4a22008-09-23 21:23:19 -07001063 if (!len) {
1064 if (dev->ifalias) {
1065 kfree(dev->ifalias);
1066 dev->ifalias = NULL;
1067 }
1068 return 0;
1069 }
1070
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001071 dev->ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001072 if (!dev->ifalias)
1073 return -ENOMEM;
1074
1075 strlcpy(dev->ifalias, alias, len+1);
1076 return len;
1077}
1078
1079
1080/**
Stephen Hemminger3041a062006-05-26 13:25:24 -07001081 * netdev_features_change - device changes features
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001082 * @dev: device to cause notification
1083 *
1084 * Called to indicate a device has changed features.
1085 */
1086void netdev_features_change(struct net_device *dev)
1087{
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001088 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001089}
1090EXPORT_SYMBOL(netdev_features_change);
1091
1092/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07001093 * netdev_state_change - device changes state
1094 * @dev: device to cause notification
1095 *
1096 * Called to indicate a device has changed state. This function calls
1097 * the notifier chains for netdev_chain and sends a NEWLINK message
1098 * to the routing socket.
1099 */
1100void netdev_state_change(struct net_device *dev)
1101{
1102 if (dev->flags & IFF_UP) {
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001103 call_netdevice_notifiers(NETDEV_CHANGE, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001104 rtmsg_ifinfo(RTM_NEWLINK, dev, 0);
1105 }
1106}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001107EXPORT_SYMBOL(netdev_state_change);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001108
Jiri Pirko3ca5b402010-03-10 10:29:35 +00001109int netdev_bonding_change(struct net_device *dev, unsigned long event)
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001110{
Jiri Pirko3ca5b402010-03-10 10:29:35 +00001111 return call_netdevice_notifiers(event, dev);
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001112}
1113EXPORT_SYMBOL(netdev_bonding_change);
1114
Linus Torvalds1da177e2005-04-16 15:20:36 -07001115/**
1116 * dev_load - load a network module
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07001117 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07001118 * @name: name of interface
1119 *
1120 * If a network interface is not present and the process has suitable
1121 * privileges this function loads the module. If module loading is not
1122 * available in this kernel then it becomes a nop.
1123 */
1124
Eric W. Biederman881d9662007-09-17 11:56:21 -07001125void dev_load(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001126{
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001127 struct net_device *dev;
Vasiliy Kulikov8909c9a2011-03-02 00:33:13 +03001128 int no_module;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001129
Eric Dumazet72c95282009-10-30 07:11:27 +00001130 rcu_read_lock();
1131 dev = dev_get_by_name_rcu(net, name);
1132 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001133
Vasiliy Kulikov8909c9a2011-03-02 00:33:13 +03001134 no_module = !dev;
1135 if (no_module && capable(CAP_NET_ADMIN))
1136 no_module = request_module("netdev-%s", name);
1137 if (no_module && capable(CAP_SYS_MODULE)) {
1138 if (!request_module("%s", name))
1139 pr_err("Loading kernel module for a network device "
1140"with CAP_SYS_MODULE (deprecated). Use CAP_NET_ADMIN and alias netdev-%s "
1141"instead\n", name);
1142 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001143}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001144EXPORT_SYMBOL(dev_load);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001145
Patrick McHardybd380812010-02-26 06:34:53 +00001146static int __dev_open(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001147{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001148 const struct net_device_ops *ops = dev->netdev_ops;
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001149 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001150
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001151 ASSERT_RTNL();
1152
Linus Torvalds1da177e2005-04-16 15:20:36 -07001153 if (!netif_device_present(dev))
1154 return -ENODEV;
1155
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001156 ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1157 ret = notifier_to_errno(ret);
1158 if (ret)
1159 return ret;
1160
Linus Torvalds1da177e2005-04-16 15:20:36 -07001161 set_bit(__LINK_STATE_START, &dev->state);
Jeff Garzikbada3392007-10-23 20:19:37 -07001162
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001163 if (ops->ndo_validate_addr)
1164 ret = ops->ndo_validate_addr(dev);
Jeff Garzikbada3392007-10-23 20:19:37 -07001165
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001166 if (!ret && ops->ndo_open)
1167 ret = ops->ndo_open(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001168
Jeff Garzikbada3392007-10-23 20:19:37 -07001169 if (ret)
1170 clear_bit(__LINK_STATE_START, &dev->state);
1171 else {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001172 dev->flags |= IFF_UP;
David S. Millerb4bd07c2009-02-06 22:06:43 -08001173 net_dmaengine_get();
Patrick McHardy4417da62007-06-27 01:28:10 -07001174 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001175 dev_activate(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001176 }
Jeff Garzikbada3392007-10-23 20:19:37 -07001177
Linus Torvalds1da177e2005-04-16 15:20:36 -07001178 return ret;
1179}
Patrick McHardybd380812010-02-26 06:34:53 +00001180
1181/**
1182 * dev_open - prepare an interface for use.
1183 * @dev: device to open
1184 *
1185 * Takes a device from down to up state. The device's private open
1186 * function is invoked and then the multicast lists are loaded. Finally
1187 * the device is moved into the up state and a %NETDEV_UP message is
1188 * sent to the netdev notifier chain.
1189 *
1190 * Calling this function on an active interface is a nop. On a failure
1191 * a negative errno code is returned.
1192 */
1193int dev_open(struct net_device *dev)
1194{
1195 int ret;
1196
Patrick McHardybd380812010-02-26 06:34:53 +00001197 if (dev->flags & IFF_UP)
1198 return 0;
1199
Patrick McHardybd380812010-02-26 06:34:53 +00001200 ret = __dev_open(dev);
1201 if (ret < 0)
1202 return ret;
1203
Patrick McHardybd380812010-02-26 06:34:53 +00001204 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
1205 call_netdevice_notifiers(NETDEV_UP, dev);
1206
1207 return ret;
1208}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001209EXPORT_SYMBOL(dev_open);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001210
Octavian Purdila44345722010-12-13 12:44:07 +00001211static int __dev_close_many(struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001212{
Octavian Purdila44345722010-12-13 12:44:07 +00001213 struct net_device *dev;
Patrick McHardybd380812010-02-26 06:34:53 +00001214
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001215 ASSERT_RTNL();
David S. Miller9d5010d2007-09-12 14:33:25 +02001216 might_sleep();
1217
Octavian Purdila44345722010-12-13 12:44:07 +00001218 list_for_each_entry(dev, head, unreg_list) {
Octavian Purdila44345722010-12-13 12:44:07 +00001219 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001220
Octavian Purdila44345722010-12-13 12:44:07 +00001221 clear_bit(__LINK_STATE_START, &dev->state);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001222
Octavian Purdila44345722010-12-13 12:44:07 +00001223 /* Synchronize to scheduled poll. We cannot touch poll list, it
1224 * can be even on different cpu. So just clear netif_running().
1225 *
1226 * dev->stop() will invoke napi_disable() on all of it's
1227 * napi_struct instances on this device.
1228 */
1229 smp_mb__after_clear_bit(); /* Commit netif_running(). */
1230 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001231
Octavian Purdila44345722010-12-13 12:44:07 +00001232 dev_deactivate_many(head);
1233
1234 list_for_each_entry(dev, head, unreg_list) {
1235 const struct net_device_ops *ops = dev->netdev_ops;
1236
1237 /*
1238 * Call the device specific close. This cannot fail.
1239 * Only if device is UP
1240 *
1241 * We allow it to be called even after a DETACH hot-plug
1242 * event.
1243 */
1244 if (ops->ndo_stop)
1245 ops->ndo_stop(dev);
1246
Octavian Purdila44345722010-12-13 12:44:07 +00001247 dev->flags &= ~IFF_UP;
Octavian Purdila44345722010-12-13 12:44:07 +00001248 net_dmaengine_put();
1249 }
1250
1251 return 0;
1252}
1253
1254static int __dev_close(struct net_device *dev)
1255{
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001256 int retval;
Octavian Purdila44345722010-12-13 12:44:07 +00001257 LIST_HEAD(single);
1258
1259 list_add(&dev->unreg_list, &single);
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001260 retval = __dev_close_many(&single);
1261 list_del(&single);
1262 return retval;
Octavian Purdila44345722010-12-13 12:44:07 +00001263}
1264
Eric Dumazet3fbd8752011-01-19 21:23:22 +00001265static int dev_close_many(struct list_head *head)
Octavian Purdila44345722010-12-13 12:44:07 +00001266{
1267 struct net_device *dev, *tmp;
1268 LIST_HEAD(tmp_list);
1269
1270 list_for_each_entry_safe(dev, tmp, head, unreg_list)
1271 if (!(dev->flags & IFF_UP))
1272 list_move(&dev->unreg_list, &tmp_list);
1273
1274 __dev_close_many(head);
Matti Linnanvuorid8b2a4d2008-02-12 23:10:11 -08001275
Octavian Purdila44345722010-12-13 12:44:07 +00001276 list_for_each_entry(dev, head, unreg_list) {
1277 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
1278 call_netdevice_notifiers(NETDEV_DOWN, dev);
1279 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001280
Octavian Purdila44345722010-12-13 12:44:07 +00001281 /* rollback_registered_many needs the complete original list */
1282 list_splice(&tmp_list, head);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001283 return 0;
1284}
Patrick McHardybd380812010-02-26 06:34:53 +00001285
1286/**
1287 * dev_close - shutdown an interface.
1288 * @dev: device to shutdown
1289 *
1290 * This function moves an active device into down state. A
1291 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1292 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1293 * chain.
1294 */
1295int dev_close(struct net_device *dev)
1296{
Eric Dumazete14a5992011-05-10 12:26:06 -07001297 if (dev->flags & IFF_UP) {
1298 LIST_HEAD(single);
Patrick McHardybd380812010-02-26 06:34:53 +00001299
Eric Dumazete14a5992011-05-10 12:26:06 -07001300 list_add(&dev->unreg_list, &single);
1301 dev_close_many(&single);
1302 list_del(&single);
1303 }
Patrick McHardybd380812010-02-26 06:34:53 +00001304 return 0;
1305}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001306EXPORT_SYMBOL(dev_close);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001307
1308
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001309/**
1310 * dev_disable_lro - disable Large Receive Offload on a device
1311 * @dev: device
1312 *
1313 * Disable Large Receive Offload (LRO) on a net device. Must be
1314 * called under RTNL. This is needed if received packets may be
1315 * forwarded to another interface.
1316 */
1317void dev_disable_lro(struct net_device *dev)
1318{
Michał Mirosław27660512011-03-18 16:56:34 +00001319 u32 flags;
1320
Neil Hormanf11970e2011-05-24 08:31:09 +00001321 /*
1322 * If we're trying to disable lro on a vlan device
1323 * use the underlying physical device instead
1324 */
1325 if (is_vlan_dev(dev))
1326 dev = vlan_dev_real_dev(dev);
1327
Michał Mirosław27660512011-03-18 16:56:34 +00001328 if (dev->ethtool_ops && dev->ethtool_ops->get_flags)
1329 flags = dev->ethtool_ops->get_flags(dev);
1330 else
1331 flags = ethtool_op_get_flags(dev);
1332
1333 if (!(flags & ETH_FLAG_LRO))
1334 return;
1335
1336 __ethtool_set_flags(dev, flags & ~ETH_FLAG_LRO);
Michał Mirosław22d59692011-04-21 12:42:15 +00001337 if (unlikely(dev->features & NETIF_F_LRO))
1338 netdev_WARN(dev, "failed to disable LRO!\n");
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001339}
1340EXPORT_SYMBOL(dev_disable_lro);
1341
1342
Eric W. Biederman881d9662007-09-17 11:56:21 -07001343static int dev_boot_phase = 1;
1344
Linus Torvalds1da177e2005-04-16 15:20:36 -07001345/**
1346 * register_netdevice_notifier - register a network notifier block
1347 * @nb: notifier
1348 *
1349 * Register a notifier to be called when network device events occur.
1350 * The notifier passed is linked into the kernel structures and must
1351 * not be reused until it has been unregistered. A negative errno code
1352 * is returned on a failure.
1353 *
1354 * When registered all registration and up events are replayed
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001355 * to the new notifier to allow device to have a race free
Linus Torvalds1da177e2005-04-16 15:20:36 -07001356 * view of the network device list.
1357 */
1358
1359int register_netdevice_notifier(struct notifier_block *nb)
1360{
1361 struct net_device *dev;
Herbert Xufcc5a032007-07-30 17:03:38 -07001362 struct net_device *last;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001363 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001364 int err;
1365
1366 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001367 err = raw_notifier_chain_register(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001368 if (err)
1369 goto unlock;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001370 if (dev_boot_phase)
1371 goto unlock;
1372 for_each_net(net) {
1373 for_each_netdev(net, dev) {
1374 err = nb->notifier_call(nb, NETDEV_REGISTER, dev);
1375 err = notifier_to_errno(err);
1376 if (err)
1377 goto rollback;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001378
Eric W. Biederman881d9662007-09-17 11:56:21 -07001379 if (!(dev->flags & IFF_UP))
1380 continue;
Herbert Xufcc5a032007-07-30 17:03:38 -07001381
Eric W. Biederman881d9662007-09-17 11:56:21 -07001382 nb->notifier_call(nb, NETDEV_UP, dev);
1383 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001384 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001385
1386unlock:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001387 rtnl_unlock();
1388 return err;
Herbert Xufcc5a032007-07-30 17:03:38 -07001389
1390rollback:
1391 last = dev;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001392 for_each_net(net) {
1393 for_each_netdev(net, dev) {
1394 if (dev == last)
1395 break;
Herbert Xufcc5a032007-07-30 17:03:38 -07001396
Eric W. Biederman881d9662007-09-17 11:56:21 -07001397 if (dev->flags & IFF_UP) {
1398 nb->notifier_call(nb, NETDEV_GOING_DOWN, dev);
1399 nb->notifier_call(nb, NETDEV_DOWN, dev);
1400 }
1401 nb->notifier_call(nb, NETDEV_UNREGISTER, dev);
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00001402 nb->notifier_call(nb, NETDEV_UNREGISTER_BATCH, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001403 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001404 }
Pavel Emelyanovc67625a2007-11-14 15:53:16 -08001405
1406 raw_notifier_chain_unregister(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001407 goto unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001408}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001409EXPORT_SYMBOL(register_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001410
1411/**
1412 * unregister_netdevice_notifier - unregister a network notifier block
1413 * @nb: notifier
1414 *
1415 * Unregister a notifier previously registered by
1416 * register_netdevice_notifier(). The notifier is unlinked into the
1417 * kernel structures and may then be reused. A negative errno code
1418 * is returned on a failure.
1419 */
1420
1421int unregister_netdevice_notifier(struct notifier_block *nb)
1422{
Herbert Xu9f514952006-03-25 01:24:25 -08001423 int err;
1424
1425 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001426 err = raw_notifier_chain_unregister(&netdev_chain, nb);
Herbert Xu9f514952006-03-25 01:24:25 -08001427 rtnl_unlock();
1428 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001429}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001430EXPORT_SYMBOL(unregister_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001431
1432/**
1433 * call_netdevice_notifiers - call all network notifier blocks
1434 * @val: value passed unmodified to notifier function
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07001435 * @dev: net_device pointer passed unmodified to notifier function
Linus Torvalds1da177e2005-04-16 15:20:36 -07001436 *
1437 * Call all network notifier blocks. Parameters and return value
Alan Sternf07d5b92006-05-09 15:23:03 -07001438 * are as for raw_notifier_call_chain().
Linus Torvalds1da177e2005-04-16 15:20:36 -07001439 */
1440
Eric W. Biedermanad7379d2007-09-16 15:33:32 -07001441int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001442{
Jiri Pirkoab930472010-04-20 01:45:37 -07001443 ASSERT_RTNL();
Eric W. Biedermanad7379d2007-09-16 15:33:32 -07001444 return raw_notifier_call_chain(&netdev_chain, val, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001445}
stephen hemmingeredf947f2011-03-24 13:24:01 +00001446EXPORT_SYMBOL(call_netdevice_notifiers);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001447
1448/* When > 0 there are consumers of rx skb time stamps */
1449static atomic_t netstamp_needed = ATOMIC_INIT(0);
1450
1451void net_enable_timestamp(void)
1452{
1453 atomic_inc(&netstamp_needed);
1454}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001455EXPORT_SYMBOL(net_enable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001456
1457void net_disable_timestamp(void)
1458{
1459 atomic_dec(&netstamp_needed);
1460}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001461EXPORT_SYMBOL(net_disable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001462
Eric Dumazet3b098e22010-05-15 23:57:10 -07001463static inline void net_timestamp_set(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001464{
1465 if (atomic_read(&netstamp_needed))
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001466 __net_timestamp(skb);
Eric Dumazetb7aa0bf2007-04-19 16:16:32 -07001467 else
1468 skb->tstamp.tv64 = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001469}
1470
Eric Dumazet3b098e22010-05-15 23:57:10 -07001471static inline void net_timestamp_check(struct sk_buff *skb)
1472{
1473 if (!skb->tstamp.tv64 && atomic_read(&netstamp_needed))
1474 __net_timestamp(skb);
1475}
1476
Daniel Lezcano79b569f2011-03-30 02:42:17 -07001477static inline bool is_skb_forwardable(struct net_device *dev,
1478 struct sk_buff *skb)
1479{
1480 unsigned int len;
1481
1482 if (!(dev->flags & IFF_UP))
1483 return false;
1484
1485 len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
1486 if (skb->len <= len)
1487 return true;
1488
1489 /* if TSO is enabled, we don't care about the length as the packet
1490 * could be forwarded without being segmented before
1491 */
1492 if (skb_is_gso(skb))
1493 return true;
1494
1495 return false;
1496}
1497
Arnd Bergmann44540962009-11-26 06:07:08 +00001498/**
1499 * dev_forward_skb - loopback an skb to another netif
1500 *
1501 * @dev: destination network device
1502 * @skb: buffer to forward
1503 *
1504 * return values:
1505 * NET_RX_SUCCESS (no congestion)
Eric Dumazet6ec82562010-05-06 00:53:53 -07001506 * NET_RX_DROP (packet was dropped, but freed)
Arnd Bergmann44540962009-11-26 06:07:08 +00001507 *
1508 * dev_forward_skb can be used for injecting an skb from the
1509 * start_xmit function of one device into the receive queue
1510 * of another device.
1511 *
1512 * The receiving device may be in another namespace, so
1513 * we have to clear all information in the skb that could
1514 * impact namespace isolation.
1515 */
1516int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1517{
1518 skb_orphan(skb);
Ben Greearc736eef2010-07-22 09:54:47 +00001519 nf_reset(skb);
Arnd Bergmann44540962009-11-26 06:07:08 +00001520
Daniel Lezcano79b569f2011-03-30 02:42:17 -07001521 if (unlikely(!is_skb_forwardable(dev, skb))) {
Eric Dumazetcaf586e2010-09-30 21:06:55 +00001522 atomic_long_inc(&dev->rx_dropped);
Eric Dumazet6ec82562010-05-06 00:53:53 -07001523 kfree_skb(skb);
Arnd Bergmann44540962009-11-26 06:07:08 +00001524 return NET_RX_DROP;
Eric Dumazet6ec82562010-05-06 00:53:53 -07001525 }
Arnd Bergmann8a83a002010-01-30 12:23:03 +00001526 skb_set_dev(skb, dev);
Arnd Bergmann44540962009-11-26 06:07:08 +00001527 skb->tstamp.tv64 = 0;
1528 skb->pkt_type = PACKET_HOST;
1529 skb->protocol = eth_type_trans(skb, dev);
Arnd Bergmann44540962009-11-26 06:07:08 +00001530 return netif_rx(skb);
1531}
1532EXPORT_SYMBOL_GPL(dev_forward_skb);
1533
Changli Gao71d9dec2010-12-15 19:57:25 +00001534static inline int deliver_skb(struct sk_buff *skb,
1535 struct packet_type *pt_prev,
1536 struct net_device *orig_dev)
1537{
1538 atomic_inc(&skb->users);
1539 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1540}
1541
Linus Torvalds1da177e2005-04-16 15:20:36 -07001542/*
1543 * Support routine. Sends outgoing frames to any network
1544 * taps currently in use.
1545 */
1546
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001547static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001548{
1549 struct packet_type *ptype;
Changli Gao71d9dec2010-12-15 19:57:25 +00001550 struct sk_buff *skb2 = NULL;
1551 struct packet_type *pt_prev = NULL;
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001552
Linus Torvalds1da177e2005-04-16 15:20:36 -07001553 rcu_read_lock();
1554 list_for_each_entry_rcu(ptype, &ptype_all, list) {
1555 /* Never send packets back to the socket
1556 * they originated from - MvS (miquels@drinkel.ow.org)
1557 */
1558 if ((ptype->dev == dev || !ptype->dev) &&
1559 (ptype->af_packet_priv == NULL ||
1560 (struct sock *)ptype->af_packet_priv != skb->sk)) {
Changli Gao71d9dec2010-12-15 19:57:25 +00001561 if (pt_prev) {
1562 deliver_skb(skb2, pt_prev, skb->dev);
1563 pt_prev = ptype;
1564 continue;
1565 }
1566
1567 skb2 = skb_clone(skb, GFP_ATOMIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001568 if (!skb2)
1569 break;
1570
Eric Dumazet70978182010-12-20 21:22:51 +00001571 net_timestamp_set(skb2);
1572
Linus Torvalds1da177e2005-04-16 15:20:36 -07001573 /* skb->nh should be correctly
1574 set by sender, so that the second statement is
1575 just protection against buggy protocols.
1576 */
Arnaldo Carvalho de Melo459a98e2007-03-19 15:30:44 -07001577 skb_reset_mac_header(skb2);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001578
Arnaldo Carvalho de Melod56f90a2007-04-10 20:50:43 -07001579 if (skb_network_header(skb2) < skb2->data ||
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -07001580 skb2->network_header > skb2->tail) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001581 if (net_ratelimit())
1582 printk(KERN_CRIT "protocol %04x is "
1583 "buggy, dev %s\n",
Sebastian Andrzej Siewior70777d02010-06-30 10:39:19 -07001584 ntohs(skb2->protocol),
1585 dev->name);
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -07001586 skb_reset_network_header(skb2);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001587 }
1588
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -07001589 skb2->transport_header = skb2->network_header;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001590 skb2->pkt_type = PACKET_OUTGOING;
Changli Gao71d9dec2010-12-15 19:57:25 +00001591 pt_prev = ptype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001592 }
1593 }
Changli Gao71d9dec2010-12-15 19:57:25 +00001594 if (pt_prev)
1595 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001596 rcu_read_unlock();
1597}
1598
John Fastabend4f57c082011-01-17 08:06:04 +00001599/* netif_setup_tc - Handle tc mappings on real_num_tx_queues change
1600 * @dev: Network device
1601 * @txq: number of queues available
1602 *
1603 * If real_num_tx_queues is changed the tc mappings may no longer be
1604 * valid. To resolve this verify the tc mapping remains valid and if
1605 * not NULL the mapping. With no priorities mapping to this
1606 * offset/count pair it will no longer be used. In the worst case TC0
1607 * is invalid nothing can be done so disable priority mappings. If is
1608 * expected that drivers will fix this mapping if they can before
1609 * calling netif_set_real_num_tx_queues.
1610 */
Eric Dumazetbb134d22011-01-20 19:18:08 +00001611static void netif_setup_tc(struct net_device *dev, unsigned int txq)
John Fastabend4f57c082011-01-17 08:06:04 +00001612{
1613 int i;
1614 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
1615
1616 /* If TC0 is invalidated disable TC mapping */
1617 if (tc->offset + tc->count > txq) {
1618 pr_warning("Number of in use tx queues changed "
1619 "invalidating tc mappings. Priority "
1620 "traffic classification disabled!\n");
1621 dev->num_tc = 0;
1622 return;
1623 }
1624
1625 /* Invalidated prio to tc mappings set to TC0 */
1626 for (i = 1; i < TC_BITMASK + 1; i++) {
1627 int q = netdev_get_prio_tc_map(dev, i);
1628
1629 tc = &dev->tc_to_txq[q];
1630 if (tc->offset + tc->count > txq) {
1631 pr_warning("Number of in use tx queues "
1632 "changed. Priority %i to tc "
1633 "mapping %i is no longer valid "
1634 "setting map to 0\n",
1635 i, q);
1636 netdev_set_prio_tc_map(dev, i, 0);
1637 }
1638 }
1639}
1640
John Fastabendf0796d52010-07-01 13:21:57 +00001641/*
1642 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
1643 * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
1644 */
Tom Herberte6484932010-10-18 18:04:39 +00001645int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
John Fastabendf0796d52010-07-01 13:21:57 +00001646{
Tom Herbert1d24eb42010-11-21 13:17:27 +00001647 int rc;
1648
Tom Herberte6484932010-10-18 18:04:39 +00001649 if (txq < 1 || txq > dev->num_tx_queues)
1650 return -EINVAL;
John Fastabendf0796d52010-07-01 13:21:57 +00001651
Ben Hutchings5c565802011-02-15 19:39:21 +00001652 if (dev->reg_state == NETREG_REGISTERED ||
1653 dev->reg_state == NETREG_UNREGISTERING) {
Tom Herberte6484932010-10-18 18:04:39 +00001654 ASSERT_RTNL();
1655
Tom Herbert1d24eb42010-11-21 13:17:27 +00001656 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
1657 txq);
Tom Herbertbf264142010-11-26 08:36:09 +00001658 if (rc)
1659 return rc;
1660
John Fastabend4f57c082011-01-17 08:06:04 +00001661 if (dev->num_tc)
1662 netif_setup_tc(dev, txq);
1663
Tom Herberte6484932010-10-18 18:04:39 +00001664 if (txq < dev->real_num_tx_queues)
1665 qdisc_reset_all_tx_gt(dev, txq);
John Fastabendf0796d52010-07-01 13:21:57 +00001666 }
Tom Herberte6484932010-10-18 18:04:39 +00001667
1668 dev->real_num_tx_queues = txq;
1669 return 0;
John Fastabendf0796d52010-07-01 13:21:57 +00001670}
1671EXPORT_SYMBOL(netif_set_real_num_tx_queues);
Denis Vlasenko56079432006-03-29 15:57:29 -08001672
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001673#ifdef CONFIG_RPS
1674/**
1675 * netif_set_real_num_rx_queues - set actual number of RX queues used
1676 * @dev: Network device
1677 * @rxq: Actual number of RX queues
1678 *
1679 * This must be called either with the rtnl_lock held or before
1680 * registration of the net device. Returns 0 on success, or a
Ben Hutchings4e7f7952010-10-08 10:33:39 -07001681 * negative error code. If called before registration, it always
1682 * succeeds.
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001683 */
1684int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
1685{
1686 int rc;
1687
Tom Herbertbd25fa72010-10-18 18:00:16 +00001688 if (rxq < 1 || rxq > dev->num_rx_queues)
1689 return -EINVAL;
1690
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001691 if (dev->reg_state == NETREG_REGISTERED) {
1692 ASSERT_RTNL();
1693
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001694 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
1695 rxq);
1696 if (rc)
1697 return rc;
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001698 }
1699
1700 dev->real_num_rx_queues = rxq;
1701 return 0;
1702}
1703EXPORT_SYMBOL(netif_set_real_num_rx_queues);
1704#endif
1705
Jarek Poplawskidef82a12008-08-17 21:54:43 -07001706static inline void __netif_reschedule(struct Qdisc *q)
1707{
1708 struct softnet_data *sd;
1709 unsigned long flags;
1710
1711 local_irq_save(flags);
1712 sd = &__get_cpu_var(softnet_data);
Changli Gaoa9cbd582010-04-26 23:06:24 +00001713 q->next_sched = NULL;
1714 *sd->output_queue_tailp = q;
1715 sd->output_queue_tailp = &q->next_sched;
Jarek Poplawskidef82a12008-08-17 21:54:43 -07001716 raise_softirq_irqoff(NET_TX_SOFTIRQ);
1717 local_irq_restore(flags);
1718}
1719
David S. Miller37437bb2008-07-16 02:15:04 -07001720void __netif_schedule(struct Qdisc *q)
Denis Vlasenko56079432006-03-29 15:57:29 -08001721{
Jarek Poplawskidef82a12008-08-17 21:54:43 -07001722 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
1723 __netif_reschedule(q);
Denis Vlasenko56079432006-03-29 15:57:29 -08001724}
1725EXPORT_SYMBOL(__netif_schedule);
1726
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001727void dev_kfree_skb_irq(struct sk_buff *skb)
Denis Vlasenko56079432006-03-29 15:57:29 -08001728{
David S. Miller3578b0c2010-08-03 00:24:04 -07001729 if (atomic_dec_and_test(&skb->users)) {
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001730 struct softnet_data *sd;
1731 unsigned long flags;
Denis Vlasenko56079432006-03-29 15:57:29 -08001732
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001733 local_irq_save(flags);
1734 sd = &__get_cpu_var(softnet_data);
1735 skb->next = sd->completion_queue;
1736 sd->completion_queue = skb;
1737 raise_softirq_irqoff(NET_TX_SOFTIRQ);
1738 local_irq_restore(flags);
1739 }
Denis Vlasenko56079432006-03-29 15:57:29 -08001740}
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001741EXPORT_SYMBOL(dev_kfree_skb_irq);
Denis Vlasenko56079432006-03-29 15:57:29 -08001742
1743void dev_kfree_skb_any(struct sk_buff *skb)
1744{
1745 if (in_irq() || irqs_disabled())
1746 dev_kfree_skb_irq(skb);
1747 else
1748 dev_kfree_skb(skb);
1749}
1750EXPORT_SYMBOL(dev_kfree_skb_any);
1751
1752
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001753/**
1754 * netif_device_detach - mark device as removed
1755 * @dev: network device
1756 *
1757 * Mark device as removed from system and therefore no longer available.
1758 */
Denis Vlasenko56079432006-03-29 15:57:29 -08001759void netif_device_detach(struct net_device *dev)
1760{
1761 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
1762 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00001763 netif_tx_stop_all_queues(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08001764 }
1765}
1766EXPORT_SYMBOL(netif_device_detach);
1767
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001768/**
1769 * netif_device_attach - mark device as attached
1770 * @dev: network device
1771 *
1772 * Mark device as attached from system and restart if needed.
1773 */
Denis Vlasenko56079432006-03-29 15:57:29 -08001774void netif_device_attach(struct net_device *dev)
1775{
1776 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
1777 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00001778 netif_tx_wake_all_queues(dev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001779 __netdev_watchdog_up(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08001780 }
1781}
1782EXPORT_SYMBOL(netif_device_attach);
1783
Arnd Bergmann8a83a002010-01-30 12:23:03 +00001784/**
1785 * skb_dev_set -- assign a new device to a buffer
1786 * @skb: buffer for the new device
1787 * @dev: network device
1788 *
1789 * If an skb is owned by a device already, we have to reset
1790 * all data private to the namespace a device belongs to
1791 * before assigning it a new device.
1792 */
1793#ifdef CONFIG_NET_NS
1794void skb_set_dev(struct sk_buff *skb, struct net_device *dev)
1795{
1796 skb_dst_drop(skb);
1797 if (skb->dev && !net_eq(dev_net(skb->dev), dev_net(dev))) {
1798 secpath_reset(skb);
1799 nf_reset(skb);
1800 skb_init_secmark(skb);
1801 skb->mark = 0;
1802 skb->priority = 0;
1803 skb->nf_trace = 0;
1804 skb->ipvs_property = 0;
1805#ifdef CONFIG_NET_SCHED
1806 skb->tc_index = 0;
1807#endif
1808 }
1809 skb->dev = dev;
1810}
1811EXPORT_SYMBOL(skb_set_dev);
1812#endif /* CONFIG_NET_NS */
1813
Linus Torvalds1da177e2005-04-16 15:20:36 -07001814/*
1815 * Invalidate hardware checksum when packet is to be mangled, and
1816 * complete checksum manually on outgoing path.
1817 */
Patrick McHardy84fa7932006-08-29 16:44:56 -07001818int skb_checksum_help(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001819{
Al Virod3bc23e2006-11-14 21:24:49 -08001820 __wsum csum;
Herbert Xu663ead32007-04-09 11:59:07 -07001821 int ret = 0, offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001822
Patrick McHardy84fa7932006-08-29 16:44:56 -07001823 if (skb->ip_summed == CHECKSUM_COMPLETE)
Herbert Xua430a432006-07-08 13:34:56 -07001824 goto out_set_summed;
1825
1826 if (unlikely(skb_shinfo(skb)->gso_size)) {
Herbert Xua430a432006-07-08 13:34:56 -07001827 /* Let GSO fix up the checksum. */
1828 goto out_set_summed;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001829 }
1830
Michał Mirosław55508d62010-12-14 15:24:08 +00001831 offset = skb_checksum_start_offset(skb);
Herbert Xua0308472007-10-15 01:47:15 -07001832 BUG_ON(offset >= skb_headlen(skb));
1833 csum = skb_checksum(skb, offset, skb->len - offset, 0);
1834
1835 offset += skb->csum_offset;
1836 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
1837
1838 if (skb_cloned(skb) &&
1839 !skb_clone_writable(skb, offset + sizeof(__sum16))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001840 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
1841 if (ret)
1842 goto out;
1843 }
1844
Herbert Xua0308472007-10-15 01:47:15 -07001845 *(__sum16 *)(skb->data + offset) = csum_fold(csum);
Herbert Xua430a432006-07-08 13:34:56 -07001846out_set_summed:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001847 skb->ip_summed = CHECKSUM_NONE;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001848out:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001849 return ret;
1850}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001851EXPORT_SYMBOL(skb_checksum_help);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001852
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001853/**
1854 * skb_gso_segment - Perform segmentation on skb.
1855 * @skb: buffer to segment
Herbert Xu576a30e2006-06-27 13:22:38 -07001856 * @features: features for the output path (see dev->features)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001857 *
1858 * This function segments the given skb and returns a list of segments.
Herbert Xu576a30e2006-06-27 13:22:38 -07001859 *
1860 * It may return NULL if the skb requires no segmentation. This is
1861 * only possible when GSO is used for verifying header integrity.
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001862 */
Michał Mirosław04ed3e72011-01-24 15:32:47 -08001863struct sk_buff *skb_gso_segment(struct sk_buff *skb, u32 features)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001864{
1865 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
1866 struct packet_type *ptype;
Al Viro252e3342006-11-14 20:48:11 -08001867 __be16 type = skb->protocol;
Jesse Grossc8d5bcd2010-10-29 12:14:54 +00001868 int vlan_depth = ETH_HLEN;
Herbert Xua430a432006-07-08 13:34:56 -07001869 int err;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001870
Jesse Grossc8d5bcd2010-10-29 12:14:54 +00001871 while (type == htons(ETH_P_8021Q)) {
1872 struct vlan_hdr *vh;
Jesse Gross7b9c6092010-10-20 13:56:04 +00001873
Jesse Grossc8d5bcd2010-10-29 12:14:54 +00001874 if (unlikely(!pskb_may_pull(skb, vlan_depth + VLAN_HLEN)))
Jesse Gross7b9c6092010-10-20 13:56:04 +00001875 return ERR_PTR(-EINVAL);
1876
Jesse Grossc8d5bcd2010-10-29 12:14:54 +00001877 vh = (struct vlan_hdr *)(skb->data + vlan_depth);
1878 type = vh->h_vlan_encapsulated_proto;
1879 vlan_depth += VLAN_HLEN;
Jesse Gross7b9c6092010-10-20 13:56:04 +00001880 }
1881
Arnaldo Carvalho de Melo459a98e2007-03-19 15:30:44 -07001882 skb_reset_mac_header(skb);
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -07001883 skb->mac_len = skb->network_header - skb->mac_header;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001884 __skb_pull(skb, skb->mac_len);
1885
Herbert Xu67fd1a72009-01-19 16:26:44 -08001886 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
1887 struct net_device *dev = skb->dev;
1888 struct ethtool_drvinfo info = {};
1889
1890 if (dev && dev->ethtool_ops && dev->ethtool_ops->get_drvinfo)
1891 dev->ethtool_ops->get_drvinfo(dev, &info);
1892
Joe Perchesb194a362010-10-30 11:08:52 +00001893 WARN(1, "%s: caps=(0x%lx, 0x%lx) len=%d data_len=%d ip_summed=%d\n",
Herbert Xu67fd1a72009-01-19 16:26:44 -08001894 info.driver, dev ? dev->features : 0L,
1895 skb->sk ? skb->sk->sk_route_caps : 0L,
1896 skb->len, skb->data_len, skb->ip_summed);
1897
Herbert Xua430a432006-07-08 13:34:56 -07001898 if (skb_header_cloned(skb) &&
1899 (err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))
1900 return ERR_PTR(err);
1901 }
1902
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001903 rcu_read_lock();
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08001904 list_for_each_entry_rcu(ptype,
1905 &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001906 if (ptype->type == type && !ptype->dev && ptype->gso_segment) {
Patrick McHardy84fa7932006-08-29 16:44:56 -07001907 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
Herbert Xua430a432006-07-08 13:34:56 -07001908 err = ptype->gso_send_check(skb);
1909 segs = ERR_PTR(err);
1910 if (err || skb_gso_ok(skb, features))
1911 break;
Arnaldo Carvalho de Melod56f90a2007-04-10 20:50:43 -07001912 __skb_push(skb, (skb->data -
1913 skb_network_header(skb)));
Herbert Xua430a432006-07-08 13:34:56 -07001914 }
Herbert Xu576a30e2006-06-27 13:22:38 -07001915 segs = ptype->gso_segment(skb, features);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001916 break;
1917 }
1918 }
1919 rcu_read_unlock();
1920
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -07001921 __skb_push(skb, skb->data - skb_mac_header(skb));
Herbert Xu576a30e2006-06-27 13:22:38 -07001922
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001923 return segs;
1924}
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001925EXPORT_SYMBOL(skb_gso_segment);
1926
Herbert Xufb286bb2005-11-10 13:01:24 -08001927/* Take action when hardware reception checksum errors are detected. */
1928#ifdef CONFIG_BUG
1929void netdev_rx_csum_fault(struct net_device *dev)
1930{
1931 if (net_ratelimit()) {
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001932 printk(KERN_ERR "%s: hw csum failure.\n",
Stephen Hemminger246a4212005-12-08 15:21:39 -08001933 dev ? dev->name : "<unknown>");
Herbert Xufb286bb2005-11-10 13:01:24 -08001934 dump_stack();
1935 }
1936}
1937EXPORT_SYMBOL(netdev_rx_csum_fault);
1938#endif
1939
Linus Torvalds1da177e2005-04-16 15:20:36 -07001940/* Actually, we should eliminate this check as soon as we know, that:
1941 * 1. IOMMU is present and allows to map all the memory.
1942 * 2. No high memory really exists on this machine.
1943 */
1944
Eric Dumazet9092c652010-04-02 13:34:49 -07001945static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001946{
Herbert Xu3d3a8532006-06-27 13:33:10 -07001947#ifdef CONFIG_HIGHMEM
Linus Torvalds1da177e2005-04-16 15:20:36 -07001948 int i;
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00001949 if (!(dev->features & NETIF_F_HIGHDMA)) {
1950 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
1951 if (PageHighMem(skb_shinfo(skb)->frags[i].page))
1952 return 1;
1953 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001954
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00001955 if (PCI_DMA_BUS_IS_PHYS) {
1956 struct device *pdev = dev->dev.parent;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001957
Eric Dumazet9092c652010-04-02 13:34:49 -07001958 if (!pdev)
1959 return 0;
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00001960 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1961 dma_addr_t addr = page_to_phys(skb_shinfo(skb)->frags[i].page);
1962 if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
1963 return 1;
1964 }
1965 }
Herbert Xu3d3a8532006-06-27 13:33:10 -07001966#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001967 return 0;
1968}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001969
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001970struct dev_gso_cb {
1971 void (*destructor)(struct sk_buff *skb);
1972};
1973
1974#define DEV_GSO_CB(skb) ((struct dev_gso_cb *)(skb)->cb)
1975
1976static void dev_gso_skb_destructor(struct sk_buff *skb)
1977{
1978 struct dev_gso_cb *cb;
1979
1980 do {
1981 struct sk_buff *nskb = skb->next;
1982
1983 skb->next = nskb->next;
1984 nskb->next = NULL;
1985 kfree_skb(nskb);
1986 } while (skb->next);
1987
1988 cb = DEV_GSO_CB(skb);
1989 if (cb->destructor)
1990 cb->destructor(skb);
1991}
1992
1993/**
1994 * dev_gso_segment - Perform emulated hardware segmentation on skb.
1995 * @skb: buffer to segment
Jesse Gross91ecb632011-01-09 06:23:33 +00001996 * @features: device features as applicable to this skb
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001997 *
1998 * This function segments the given skb and stores the list of segments
1999 * in skb->next.
2000 */
Jesse Gross91ecb632011-01-09 06:23:33 +00002001static int dev_gso_segment(struct sk_buff *skb, int features)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002002{
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002003 struct sk_buff *segs;
2004
Herbert Xu576a30e2006-06-27 13:22:38 -07002005 segs = skb_gso_segment(skb, features);
2006
2007 /* Verifying header integrity only. */
2008 if (!segs)
2009 return 0;
2010
Hirofumi Nakagawa801678c2008-04-29 01:03:09 -07002011 if (IS_ERR(segs))
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002012 return PTR_ERR(segs);
2013
2014 skb->next = segs;
2015 DEV_GSO_CB(skb)->destructor = skb->destructor;
2016 skb->destructor = dev_gso_skb_destructor;
2017
2018 return 0;
2019}
2020
Eric Dumazetfc6055a2010-04-16 12:18:22 +00002021/*
2022 * Try to orphan skb early, right before transmission by the device.
Oliver Hartkopp2244d072010-08-17 08:59:14 +00002023 * We cannot orphan skb if tx timestamp is requested or the sk-reference
2024 * is needed on driver level for other reasons, e.g. see net/can/raw.c
Eric Dumazetfc6055a2010-04-16 12:18:22 +00002025 */
2026static inline void skb_orphan_try(struct sk_buff *skb)
2027{
Eric Dumazet87fd3082010-07-13 05:24:20 +00002028 struct sock *sk = skb->sk;
2029
Oliver Hartkopp2244d072010-08-17 08:59:14 +00002030 if (sk && !skb_shinfo(skb)->tx_flags) {
Eric Dumazet87fd3082010-07-13 05:24:20 +00002031 /* skb_tx_hash() wont be able to get sk.
2032 * We copy sk_hash into skb->rxhash
2033 */
2034 if (!skb->rxhash)
2035 skb->rxhash = sk->sk_hash;
Eric Dumazetfc6055a2010-04-16 12:18:22 +00002036 skb_orphan(skb);
Eric Dumazet87fd3082010-07-13 05:24:20 +00002037 }
Eric Dumazetfc6055a2010-04-16 12:18:22 +00002038}
2039
Jesse Gross03634662011-01-09 06:23:35 +00002040static bool can_checksum_protocol(unsigned long features, __be16 protocol)
2041{
2042 return ((features & NETIF_F_GEN_CSUM) ||
2043 ((features & NETIF_F_V4_CSUM) &&
2044 protocol == htons(ETH_P_IP)) ||
2045 ((features & NETIF_F_V6_CSUM) &&
2046 protocol == htons(ETH_P_IPV6)) ||
2047 ((features & NETIF_F_FCOE_CRC) &&
2048 protocol == htons(ETH_P_FCOE)));
2049}
2050
Michał Mirosław04ed3e72011-01-24 15:32:47 -08002051static u32 harmonize_features(struct sk_buff *skb, __be16 protocol, u32 features)
Jesse Grossf01a5232011-01-09 06:23:31 +00002052{
Eric Dumazetd4027862011-01-19 00:51:36 +00002053 if (!can_checksum_protocol(features, protocol)) {
Jesse Grossf01a5232011-01-09 06:23:31 +00002054 features &= ~NETIF_F_ALL_CSUM;
2055 features &= ~NETIF_F_SG;
2056 } else if (illegal_highdma(skb->dev, skb)) {
2057 features &= ~NETIF_F_SG;
2058 }
2059
2060 return features;
2061}
2062
Michał Mirosław04ed3e72011-01-24 15:32:47 -08002063u32 netif_skb_features(struct sk_buff *skb)
Jesse Gross58e998c2010-10-29 12:14:55 +00002064{
2065 __be16 protocol = skb->protocol;
Michał Mirosław04ed3e72011-01-24 15:32:47 -08002066 u32 features = skb->dev->features;
Jesse Gross58e998c2010-10-29 12:14:55 +00002067
2068 if (protocol == htons(ETH_P_8021Q)) {
2069 struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
2070 protocol = veh->h_vlan_encapsulated_proto;
Jesse Grossf01a5232011-01-09 06:23:31 +00002071 } else if (!vlan_tx_tag_present(skb)) {
2072 return harmonize_features(skb, protocol, features);
2073 }
Jesse Gross58e998c2010-10-29 12:14:55 +00002074
Jesse Gross6ee400a2011-01-17 20:46:00 +00002075 features &= (skb->dev->vlan_features | NETIF_F_HW_VLAN_TX);
Jesse Grossf01a5232011-01-09 06:23:31 +00002076
2077 if (protocol != htons(ETH_P_8021Q)) {
2078 return harmonize_features(skb, protocol, features);
2079 } else {
2080 features &= NETIF_F_SG | NETIF_F_HIGHDMA | NETIF_F_FRAGLIST |
Jesse Gross6ee400a2011-01-17 20:46:00 +00002081 NETIF_F_GEN_CSUM | NETIF_F_HW_VLAN_TX;
Jesse Grossf01a5232011-01-09 06:23:31 +00002082 return harmonize_features(skb, protocol, features);
2083 }
Jesse Gross58e998c2010-10-29 12:14:55 +00002084}
Jesse Grossf01a5232011-01-09 06:23:31 +00002085EXPORT_SYMBOL(netif_skb_features);
Jesse Gross58e998c2010-10-29 12:14:55 +00002086
John Fastabend6afff0c2010-06-16 14:18:12 +00002087/*
2088 * Returns true if either:
2089 * 1. skb has frag_list and the device doesn't support FRAGLIST, or
2090 * 2. skb is fragmented and the device does not support SG, or if
2091 * at least one of fragments is in highmem and device does not
2092 * support DMA from it.
2093 */
2094static inline int skb_needs_linearize(struct sk_buff *skb,
Jesse Gross02932ce2011-01-09 06:23:34 +00002095 int features)
John Fastabend6afff0c2010-06-16 14:18:12 +00002096{
Jesse Gross02932ce2011-01-09 06:23:34 +00002097 return skb_is_nonlinear(skb) &&
2098 ((skb_has_frag_list(skb) &&
2099 !(features & NETIF_F_FRAGLIST)) ||
Jesse Grosse1e78db2010-10-29 12:14:53 +00002100 (skb_shinfo(skb)->nr_frags &&
Jesse Gross02932ce2011-01-09 06:23:34 +00002101 !(features & NETIF_F_SG)));
John Fastabend6afff0c2010-06-16 14:18:12 +00002102}
2103
David S. Millerfd2ea0a2008-07-17 01:56:23 -07002104int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
2105 struct netdev_queue *txq)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002106{
Stephen Hemminger00829822008-11-20 20:14:53 -08002107 const struct net_device_ops *ops = dev->netdev_ops;
Patrick McHardy572a9d72009-11-10 06:14:14 +00002108 int rc = NETDEV_TX_OK;
Koki Sanagiec764bf2011-05-30 21:48:34 +00002109 unsigned int skb_len;
Stephen Hemminger00829822008-11-20 20:14:53 -08002110
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002111 if (likely(!skb->next)) {
Michał Mirosław04ed3e72011-01-24 15:32:47 -08002112 u32 features;
Jesse Grossfc741212011-01-09 06:23:32 +00002113
Eric Dumazet93f154b2009-05-18 22:19:19 -07002114 /*
Lucas De Marchi25985ed2011-03-30 22:57:33 -03002115 * If device doesn't need skb->dst, release it right now while
Eric Dumazet93f154b2009-05-18 22:19:19 -07002116 * its hot in this cpu cache
2117 */
Eric Dumazetadf30902009-06-02 05:19:30 +00002118 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
2119 skb_dst_drop(skb);
2120
Eric Dumazet15c2d75f2010-12-07 00:30:37 +00002121 if (!list_empty(&ptype_all))
2122 dev_queue_xmit_nit(skb, dev);
2123
Eric Dumazetfc6055a2010-04-16 12:18:22 +00002124 skb_orphan_try(skb);
David S. Miller9ccb8972010-04-22 01:02:07 -07002125
Jesse Grossfc741212011-01-09 06:23:32 +00002126 features = netif_skb_features(skb);
2127
Jesse Gross7b9c6092010-10-20 13:56:04 +00002128 if (vlan_tx_tag_present(skb) &&
Jesse Grossfc741212011-01-09 06:23:32 +00002129 !(features & NETIF_F_HW_VLAN_TX)) {
Jesse Gross7b9c6092010-10-20 13:56:04 +00002130 skb = __vlan_put_tag(skb, vlan_tx_tag_get(skb));
2131 if (unlikely(!skb))
2132 goto out;
2133
2134 skb->vlan_tci = 0;
2135 }
2136
Jesse Grossfc741212011-01-09 06:23:32 +00002137 if (netif_needs_gso(skb, features)) {
Jesse Gross91ecb632011-01-09 06:23:33 +00002138 if (unlikely(dev_gso_segment(skb, features)))
David S. Miller9ccb8972010-04-22 01:02:07 -07002139 goto out_kfree_skb;
2140 if (skb->next)
2141 goto gso;
John Fastabend6afff0c2010-06-16 14:18:12 +00002142 } else {
Jesse Gross02932ce2011-01-09 06:23:34 +00002143 if (skb_needs_linearize(skb, features) &&
John Fastabend6afff0c2010-06-16 14:18:12 +00002144 __skb_linearize(skb))
2145 goto out_kfree_skb;
2146
2147 /* If packet is not checksummed and device does not
2148 * support checksumming for this protocol, complete
2149 * checksumming here.
2150 */
2151 if (skb->ip_summed == CHECKSUM_PARTIAL) {
Michał Mirosław55508d62010-12-14 15:24:08 +00002152 skb_set_transport_header(skb,
2153 skb_checksum_start_offset(skb));
Jesse Gross03634662011-01-09 06:23:35 +00002154 if (!(features & NETIF_F_ALL_CSUM) &&
John Fastabend6afff0c2010-06-16 14:18:12 +00002155 skb_checksum_help(skb))
2156 goto out_kfree_skb;
2157 }
David S. Miller9ccb8972010-04-22 01:02:07 -07002158 }
2159
Koki Sanagiec764bf2011-05-30 21:48:34 +00002160 skb_len = skb->len;
Patrick Ohlyac45f602009-02-12 05:03:37 +00002161 rc = ops->ndo_start_xmit(skb, dev);
Koki Sanagiec764bf2011-05-30 21:48:34 +00002162 trace_net_dev_xmit(skb, rc, dev, skb_len);
Patrick McHardyec634fe2009-07-05 19:23:38 -07002163 if (rc == NETDEV_TX_OK)
Eric Dumazet08baf562009-05-25 22:58:01 -07002164 txq_trans_update(txq);
Patrick Ohlyac45f602009-02-12 05:03:37 +00002165 return rc;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002166 }
2167
Herbert Xu576a30e2006-06-27 13:22:38 -07002168gso:
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002169 do {
2170 struct sk_buff *nskb = skb->next;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002171
2172 skb->next = nskb->next;
2173 nskb->next = NULL;
Krishna Kumar068a2de2009-12-09 20:59:58 +00002174
2175 /*
Lucas De Marchi25985ed2011-03-30 22:57:33 -03002176 * If device doesn't need nskb->dst, release it right now while
Krishna Kumar068a2de2009-12-09 20:59:58 +00002177 * its hot in this cpu cache
2178 */
2179 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
2180 skb_dst_drop(nskb);
2181
Koki Sanagiec764bf2011-05-30 21:48:34 +00002182 skb_len = nskb->len;
Stephen Hemminger00829822008-11-20 20:14:53 -08002183 rc = ops->ndo_start_xmit(nskb, dev);
Koki Sanagiec764bf2011-05-30 21:48:34 +00002184 trace_net_dev_xmit(nskb, rc, dev, skb_len);
Patrick McHardyec634fe2009-07-05 19:23:38 -07002185 if (unlikely(rc != NETDEV_TX_OK)) {
Patrick McHardy572a9d72009-11-10 06:14:14 +00002186 if (rc & ~NETDEV_TX_MASK)
2187 goto out_kfree_gso_skb;
Michael Chanf54d9e82006-06-25 23:57:04 -07002188 nskb->next = skb->next;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002189 skb->next = nskb;
2190 return rc;
2191 }
Eric Dumazet08baf562009-05-25 22:58:01 -07002192 txq_trans_update(txq);
David S. Millerfd2ea0a2008-07-17 01:56:23 -07002193 if (unlikely(netif_tx_queue_stopped(txq) && skb->next))
Michael Chanf54d9e82006-06-25 23:57:04 -07002194 return NETDEV_TX_BUSY;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002195 } while (skb->next);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002196
Patrick McHardy572a9d72009-11-10 06:14:14 +00002197out_kfree_gso_skb:
2198 if (likely(skb->next == NULL))
2199 skb->destructor = DEV_GSO_CB(skb)->destructor;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002200out_kfree_skb:
2201 kfree_skb(skb);
Jesse Gross7b9c6092010-10-20 13:56:04 +00002202out:
Patrick McHardy572a9d72009-11-10 06:14:14 +00002203 return rc;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002204}
2205
Tom Herbert0a9627f2010-03-16 08:03:29 +00002206static u32 hashrnd __read_mostly;
David S. Millerb6b2fed2008-07-21 09:48:06 -07002207
Vladislav Zolotarova3d22a62010-12-13 06:27:10 +00002208/*
2209 * Returns a Tx hash based on the given packet descriptor a Tx queues' number
2210 * to be used as a distribution range.
2211 */
2212u16 __skb_tx_hash(const struct net_device *dev, const struct sk_buff *skb,
2213 unsigned int num_tx_queues)
David S. Miller8f0f2222008-07-15 03:47:03 -07002214{
David S. Miller70192982009-01-27 16:34:47 -08002215 u32 hash;
John Fastabend4f57c082011-01-17 08:06:04 +00002216 u16 qoffset = 0;
2217 u16 qcount = num_tx_queues;
David S. Millerb6b2fed2008-07-21 09:48:06 -07002218
David S. Miller513de112009-05-03 14:43:10 -07002219 if (skb_rx_queue_recorded(skb)) {
2220 hash = skb_get_rx_queue(skb);
Vladislav Zolotarova3d22a62010-12-13 06:27:10 +00002221 while (unlikely(hash >= num_tx_queues))
2222 hash -= num_tx_queues;
David S. Miller513de112009-05-03 14:43:10 -07002223 return hash;
2224 }
Eric Dumazetec581f62009-05-01 09:05:06 -07002225
John Fastabend4f57c082011-01-17 08:06:04 +00002226 if (dev->num_tc) {
2227 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
2228 qoffset = dev->tc_to_txq[tc].offset;
2229 qcount = dev->tc_to_txq[tc].count;
2230 }
2231
Eric Dumazetec581f62009-05-01 09:05:06 -07002232 if (skb->sk && skb->sk->sk_hash)
David S. Miller70192982009-01-27 16:34:47 -08002233 hash = skb->sk->sk_hash;
Eric Dumazetec581f62009-05-01 09:05:06 -07002234 else
Eric Dumazet87fd3082010-07-13 05:24:20 +00002235 hash = (__force u16) skb->protocol ^ skb->rxhash;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002236 hash = jhash_1word(hash, hashrnd);
David S. Millerd5a9e242009-01-27 16:22:11 -08002237
John Fastabend4f57c082011-01-17 08:06:04 +00002238 return (u16) (((u64) hash * qcount) >> 32) + qoffset;
David S. Miller8f0f2222008-07-15 03:47:03 -07002239}
Vladislav Zolotarova3d22a62010-12-13 06:27:10 +00002240EXPORT_SYMBOL(__skb_tx_hash);
David S. Miller8f0f2222008-07-15 03:47:03 -07002241
Eric Dumazeted046422009-11-13 21:54:04 +00002242static inline u16 dev_cap_txqueue(struct net_device *dev, u16 queue_index)
2243{
2244 if (unlikely(queue_index >= dev->real_num_tx_queues)) {
2245 if (net_ratelimit()) {
Eric Dumazet7a161ea2010-04-08 21:26:13 +00002246 pr_warning("%s selects TX queue %d, but "
2247 "real number of TX queues is %d\n",
2248 dev->name, queue_index, dev->real_num_tx_queues);
Eric Dumazeted046422009-11-13 21:54:04 +00002249 }
2250 return 0;
2251 }
2252 return queue_index;
2253}
2254
Tom Herbert1d24eb42010-11-21 13:17:27 +00002255static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb)
2256{
Tom Herbertbf264142010-11-26 08:36:09 +00002257#ifdef CONFIG_XPS
Tom Herbert1d24eb42010-11-21 13:17:27 +00002258 struct xps_dev_maps *dev_maps;
2259 struct xps_map *map;
2260 int queue_index = -1;
2261
2262 rcu_read_lock();
2263 dev_maps = rcu_dereference(dev->xps_maps);
2264 if (dev_maps) {
2265 map = rcu_dereference(
2266 dev_maps->cpu_map[raw_smp_processor_id()]);
2267 if (map) {
2268 if (map->len == 1)
2269 queue_index = map->queues[0];
2270 else {
2271 u32 hash;
2272 if (skb->sk && skb->sk->sk_hash)
2273 hash = skb->sk->sk_hash;
2274 else
2275 hash = (__force u16) skb->protocol ^
2276 skb->rxhash;
2277 hash = jhash_1word(hash, hashrnd);
2278 queue_index = map->queues[
2279 ((u64)hash * map->len) >> 32];
2280 }
2281 if (unlikely(queue_index >= dev->real_num_tx_queues))
2282 queue_index = -1;
2283 }
2284 }
2285 rcu_read_unlock();
2286
2287 return queue_index;
2288#else
2289 return -1;
2290#endif
2291}
2292
David S. Millere8a04642008-07-17 00:34:19 -07002293static struct netdev_queue *dev_pick_tx(struct net_device *dev,
2294 struct sk_buff *skb)
2295{
Tom Herbertb0f77d02010-07-14 20:50:29 -07002296 int queue_index;
Helmut Schaadeabc772010-09-03 02:39:56 +00002297 const struct net_device_ops *ops = dev->netdev_ops;
David S. Millerfd2ea0a2008-07-17 01:56:23 -07002298
Tom Herbert3853b582010-11-21 13:17:29 +00002299 if (dev->real_num_tx_queues == 1)
2300 queue_index = 0;
2301 else if (ops->ndo_select_queue) {
Helmut Schaadeabc772010-09-03 02:39:56 +00002302 queue_index = ops->ndo_select_queue(dev, skb);
2303 queue_index = dev_cap_txqueue(dev, queue_index);
2304 } else {
2305 struct sock *sk = skb->sk;
2306 queue_index = sk_tx_queue_get(sk);
Krishna Kumara4ee3ce2009-10-19 23:50:07 +00002307
Tom Herbert3853b582010-11-21 13:17:29 +00002308 if (queue_index < 0 || skb->ooo_okay ||
2309 queue_index >= dev->real_num_tx_queues) {
2310 int old_index = queue_index;
Krishna Kumara4ee3ce2009-10-19 23:50:07 +00002311
Tom Herbert1d24eb42010-11-21 13:17:27 +00002312 queue_index = get_xps_queue(dev, skb);
2313 if (queue_index < 0)
2314 queue_index = skb_tx_hash(dev, skb);
Tom Herbert3853b582010-11-21 13:17:29 +00002315
2316 if (queue_index != old_index && sk) {
2317 struct dst_entry *dst =
2318 rcu_dereference_check(sk->sk_dst_cache, 1);
Eric Dumazet8728c542010-04-11 21:18:17 +00002319
2320 if (dst && skb_dst(skb) == dst)
2321 sk_tx_queue_set(sk, queue_index);
2322 }
Krishna Kumara4ee3ce2009-10-19 23:50:07 +00002323 }
2324 }
David S. Millereae792b2008-07-15 03:03:33 -07002325
David S. Millerfd2ea0a2008-07-17 01:56:23 -07002326 skb_set_queue_mapping(skb, queue_index);
2327 return netdev_get_tx_queue(dev, queue_index);
David S. Millere8a04642008-07-17 00:34:19 -07002328}
2329
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002330static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
2331 struct net_device *dev,
2332 struct netdev_queue *txq)
2333{
2334 spinlock_t *root_lock = qdisc_lock(q);
Eric Dumazeta2da5702011-01-20 03:48:19 +00002335 bool contended;
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002336 int rc;
2337
Eric Dumazeta2da5702011-01-20 03:48:19 +00002338 qdisc_skb_cb(skb)->pkt_len = skb->len;
2339 qdisc_calculate_pkt_len(skb, q);
Eric Dumazet79640a42010-06-02 05:09:29 -07002340 /*
2341 * Heuristic to force contended enqueues to serialize on a
2342 * separate lock before trying to get qdisc main lock.
2343 * This permits __QDISC_STATE_RUNNING owner to get the lock more often
2344 * and dequeue packets faster.
2345 */
Eric Dumazeta2da5702011-01-20 03:48:19 +00002346 contended = qdisc_is_running(q);
Eric Dumazet79640a42010-06-02 05:09:29 -07002347 if (unlikely(contended))
2348 spin_lock(&q->busylock);
2349
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002350 spin_lock(root_lock);
2351 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
2352 kfree_skb(skb);
2353 rc = NET_XMIT_DROP;
2354 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
Eric Dumazetbc135b22010-06-02 03:23:51 -07002355 qdisc_run_begin(q)) {
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002356 /*
2357 * This is a work-conserving queue; there are no old skbs
2358 * waiting to be sent out; and the qdisc is not running -
2359 * xmit the skb directly.
2360 */
Eric Dumazet7fee2262010-05-11 23:19:48 +00002361 if (!(dev->priv_flags & IFF_XMIT_DST_RELEASE))
2362 skb_dst_force(skb);
Eric Dumazetbfe0d022011-01-09 08:30:54 +00002363
Eric Dumazetbfe0d022011-01-09 08:30:54 +00002364 qdisc_bstats_update(q, skb);
2365
Eric Dumazet79640a42010-06-02 05:09:29 -07002366 if (sch_direct_xmit(skb, q, dev, txq, root_lock)) {
2367 if (unlikely(contended)) {
2368 spin_unlock(&q->busylock);
2369 contended = false;
2370 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002371 __qdisc_run(q);
Eric Dumazet79640a42010-06-02 05:09:29 -07002372 } else
Eric Dumazetbc135b22010-06-02 03:23:51 -07002373 qdisc_run_end(q);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002374
2375 rc = NET_XMIT_SUCCESS;
2376 } else {
Eric Dumazet7fee2262010-05-11 23:19:48 +00002377 skb_dst_force(skb);
Eric Dumazeta2da5702011-01-20 03:48:19 +00002378 rc = q->enqueue(skb, q) & NET_XMIT_MASK;
Eric Dumazet79640a42010-06-02 05:09:29 -07002379 if (qdisc_run_begin(q)) {
2380 if (unlikely(contended)) {
2381 spin_unlock(&q->busylock);
2382 contended = false;
2383 }
2384 __qdisc_run(q);
2385 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002386 }
2387 spin_unlock(root_lock);
Eric Dumazet79640a42010-06-02 05:09:29 -07002388 if (unlikely(contended))
2389 spin_unlock(&q->busylock);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002390 return rc;
2391}
2392
Eric Dumazet745e20f2010-09-29 13:23:09 -07002393static DEFINE_PER_CPU(int, xmit_recursion);
David S. Miller11a766c2010-10-25 12:51:55 -07002394#define RECURSION_LIMIT 10
Eric Dumazet745e20f2010-09-29 13:23:09 -07002395
Dave Jonesd29f7492008-07-22 14:09:06 -07002396/**
2397 * dev_queue_xmit - transmit a buffer
2398 * @skb: buffer to transmit
2399 *
2400 * Queue a buffer for transmission to a network device. The caller must
2401 * have set the device and priority and built the buffer before calling
2402 * this function. The function can be called from an interrupt.
2403 *
2404 * A negative errno code is returned on a failure. A success does not
2405 * guarantee the frame will be transmitted as it may be dropped due
2406 * to congestion or traffic shaping.
2407 *
2408 * -----------------------------------------------------------------------------------
2409 * I notice this method can also return errors from the queue disciplines,
2410 * including NET_XMIT_DROP, which is a positive value. So, errors can also
2411 * be positive.
2412 *
2413 * Regardless of the return value, the skb is consumed, so it is currently
2414 * difficult to retry a send to this method. (You can bump the ref count
2415 * before sending to hold a reference for retry if you are careful.)
2416 *
2417 * When calling this method, interrupts MUST be enabled. This is because
2418 * the BH enable code must have IRQs enabled so that it will not deadlock.
2419 * --BLG
2420 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002421int dev_queue_xmit(struct sk_buff *skb)
2422{
2423 struct net_device *dev = skb->dev;
David S. Millerdc2b4842008-07-08 17:18:23 -07002424 struct netdev_queue *txq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002425 struct Qdisc *q;
2426 int rc = -ENOMEM;
2427
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002428 /* Disable soft irqs for various locks below. Also
2429 * stops preemption for RCU.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002430 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002431 rcu_read_lock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002432
David S. Millereae792b2008-07-15 03:03:33 -07002433 txq = dev_pick_tx(dev, skb);
Paul E. McKenneya898def2010-02-22 17:04:49 -08002434 q = rcu_dereference_bh(txq->qdisc);
David S. Miller37437bb2008-07-16 02:15:04 -07002435
Linus Torvalds1da177e2005-04-16 15:20:36 -07002436#ifdef CONFIG_NET_CLS_ACT
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002437 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002438#endif
Koki Sanagicf66ba52010-08-23 18:45:02 +09002439 trace_net_dev_queue(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002440 if (q->enqueue) {
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002441 rc = __dev_xmit_skb(skb, q, dev, txq);
David S. Miller37437bb2008-07-16 02:15:04 -07002442 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002443 }
2444
2445 /* The device has no queue. Common case for software devices:
2446 loopback, all the sorts of tunnels...
2447
Herbert Xu932ff272006-06-09 12:20:56 -07002448 Really, it is unlikely that netif_tx_lock protection is necessary
2449 here. (f.e. loopback and IP tunnels are clean ignoring statistics
Linus Torvalds1da177e2005-04-16 15:20:36 -07002450 counters.)
2451 However, it is possible, that they rely on protection
2452 made by us here.
2453
2454 Check this and shot the lock. It is not prone from deadlocks.
2455 Either shot noqueue qdisc, it is even simpler 8)
2456 */
2457 if (dev->flags & IFF_UP) {
2458 int cpu = smp_processor_id(); /* ok because BHs are off */
2459
David S. Millerc773e842008-07-08 23:13:53 -07002460 if (txq->xmit_lock_owner != cpu) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002461
Eric Dumazet745e20f2010-09-29 13:23:09 -07002462 if (__this_cpu_read(xmit_recursion) > RECURSION_LIMIT)
2463 goto recursion_alert;
2464
David S. Millerc773e842008-07-08 23:13:53 -07002465 HARD_TX_LOCK(dev, txq, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002466
David S. Millerfd2ea0a2008-07-17 01:56:23 -07002467 if (!netif_tx_queue_stopped(txq)) {
Eric Dumazet745e20f2010-09-29 13:23:09 -07002468 __this_cpu_inc(xmit_recursion);
Patrick McHardy572a9d72009-11-10 06:14:14 +00002469 rc = dev_hard_start_xmit(skb, dev, txq);
Eric Dumazet745e20f2010-09-29 13:23:09 -07002470 __this_cpu_dec(xmit_recursion);
Patrick McHardy572a9d72009-11-10 06:14:14 +00002471 if (dev_xmit_complete(rc)) {
David S. Millerc773e842008-07-08 23:13:53 -07002472 HARD_TX_UNLOCK(dev, txq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002473 goto out;
2474 }
2475 }
David S. Millerc773e842008-07-08 23:13:53 -07002476 HARD_TX_UNLOCK(dev, txq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002477 if (net_ratelimit())
2478 printk(KERN_CRIT "Virtual device %s asks to "
2479 "queue packet!\n", dev->name);
2480 } else {
2481 /* Recursion is detected! It is possible,
Eric Dumazet745e20f2010-09-29 13:23:09 -07002482 * unfortunately
2483 */
2484recursion_alert:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002485 if (net_ratelimit())
2486 printk(KERN_CRIT "Dead loop on virtual device "
2487 "%s, fix it urgently!\n", dev->name);
2488 }
2489 }
2490
2491 rc = -ENETDOWN;
Herbert Xud4828d82006-06-22 02:28:18 -07002492 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002493
Linus Torvalds1da177e2005-04-16 15:20:36 -07002494 kfree_skb(skb);
2495 return rc;
2496out:
Herbert Xud4828d82006-06-22 02:28:18 -07002497 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002498 return rc;
2499}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002500EXPORT_SYMBOL(dev_queue_xmit);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002501
2502
2503/*=======================================================================
2504 Receiver routines
2505 =======================================================================*/
2506
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07002507int netdev_max_backlog __read_mostly = 1000;
Eric Dumazet3b098e22010-05-15 23:57:10 -07002508int netdev_tstamp_prequeue __read_mostly = 1;
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07002509int netdev_budget __read_mostly = 300;
2510int weight_p __read_mostly = 64; /* old backlog weight */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002511
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07002512/* Called with irq disabled */
2513static inline void ____napi_schedule(struct softnet_data *sd,
2514 struct napi_struct *napi)
2515{
2516 list_add_tail(&napi->poll_list, &sd->poll_list);
2517 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
2518}
2519
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002520/*
2521 * __skb_get_rxhash: calculate a flow hash based on src/dst addresses
2522 * and src/dst port numbers. Returns a non-zero hash number on success
2523 * and 0 on failure.
2524 */
2525__u32 __skb_get_rxhash(struct sk_buff *skb)
2526{
Changli Gao12fcdef2010-08-17 19:04:32 +00002527 int nhoff, hash = 0, poff;
Eric Dumazetb71d1d42011-04-22 04:53:02 +00002528 const struct ipv6hdr *ip6;
2529 const struct iphdr *ip;
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002530 u8 ip_proto;
2531 u32 addr1, addr2, ihl;
2532 union {
2533 u32 v32;
2534 u16 v16[2];
2535 } ports;
2536
2537 nhoff = skb_network_offset(skb);
2538
2539 switch (skb->protocol) {
2540 case __constant_htons(ETH_P_IP):
2541 if (!pskb_may_pull(skb, sizeof(*ip) + nhoff))
2542 goto done;
2543
Eric Dumazetb71d1d42011-04-22 04:53:02 +00002544 ip = (const struct iphdr *) (skb->data + nhoff);
Paul Gortmaker56f8a752011-06-21 20:33:34 -07002545 if (ip_is_fragment(ip))
Changli Gaodbe57752010-08-17 19:01:38 +00002546 ip_proto = 0;
2547 else
2548 ip_proto = ip->protocol;
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002549 addr1 = (__force u32) ip->saddr;
2550 addr2 = (__force u32) ip->daddr;
2551 ihl = ip->ihl;
2552 break;
2553 case __constant_htons(ETH_P_IPV6):
2554 if (!pskb_may_pull(skb, sizeof(*ip6) + nhoff))
2555 goto done;
2556
Eric Dumazetb71d1d42011-04-22 04:53:02 +00002557 ip6 = (const struct ipv6hdr *) (skb->data + nhoff);
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002558 ip_proto = ip6->nexthdr;
2559 addr1 = (__force u32) ip6->saddr.s6_addr32[3];
2560 addr2 = (__force u32) ip6->daddr.s6_addr32[3];
2561 ihl = (40 >> 2);
2562 break;
2563 default:
2564 goto done;
2565 }
2566
Changli Gao12fcdef2010-08-17 19:04:32 +00002567 ports.v32 = 0;
2568 poff = proto_ports_offset(ip_proto);
2569 if (poff >= 0) {
2570 nhoff += ihl * 4 + poff;
2571 if (pskb_may_pull(skb, nhoff + 4)) {
2572 ports.v32 = * (__force u32 *) (skb->data + nhoff);
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002573 if (ports.v16[1] < ports.v16[0])
2574 swap(ports.v16[0], ports.v16[1]);
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002575 }
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002576 }
2577
2578 /* get a consistent hash (same value on both flow directions) */
2579 if (addr2 < addr1)
2580 swap(addr1, addr2);
2581
2582 hash = jhash_3words(addr1, addr2, ports.v32, hashrnd);
2583 if (!hash)
2584 hash = 1;
2585
2586done:
2587 return hash;
2588}
2589EXPORT_SYMBOL(__skb_get_rxhash);
2590
Eric Dumazetdf334542010-03-24 19:13:54 +00002591#ifdef CONFIG_RPS
Tom Herbertfec5e652010-04-16 16:01:27 -07002592
2593/* One global table that all flow-based protocols share. */
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00002594struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
Tom Herbertfec5e652010-04-16 16:01:27 -07002595EXPORT_SYMBOL(rps_sock_flow_table);
2596
Ben Hutchingsc4454772011-01-19 11:03:53 +00002597static struct rps_dev_flow *
2598set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
2599 struct rps_dev_flow *rflow, u16 next_cpu)
2600{
2601 u16 tcpu;
2602
2603 tcpu = rflow->cpu = next_cpu;
2604 if (tcpu != RPS_NO_CPU) {
2605#ifdef CONFIG_RFS_ACCEL
2606 struct netdev_rx_queue *rxqueue;
2607 struct rps_dev_flow_table *flow_table;
2608 struct rps_dev_flow *old_rflow;
2609 u32 flow_id;
2610 u16 rxq_index;
2611 int rc;
2612
2613 /* Should we steer this flow to a different hardware queue? */
Ben Hutchings69a19ee2011-02-15 20:32:04 +00002614 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
2615 !(dev->features & NETIF_F_NTUPLE))
Ben Hutchingsc4454772011-01-19 11:03:53 +00002616 goto out;
2617 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
2618 if (rxq_index == skb_get_rx_queue(skb))
2619 goto out;
2620
2621 rxqueue = dev->_rx + rxq_index;
2622 flow_table = rcu_dereference(rxqueue->rps_flow_table);
2623 if (!flow_table)
2624 goto out;
2625 flow_id = skb->rxhash & flow_table->mask;
2626 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
2627 rxq_index, flow_id);
2628 if (rc < 0)
2629 goto out;
2630 old_rflow = rflow;
2631 rflow = &flow_table->flows[flow_id];
2632 rflow->cpu = next_cpu;
2633 rflow->filter = rc;
2634 if (old_rflow->filter == rflow->filter)
2635 old_rflow->filter = RPS_NO_FILTER;
2636 out:
2637#endif
2638 rflow->last_qtail =
2639 per_cpu(softnet_data, tcpu).input_queue_head;
2640 }
2641
2642 return rflow;
2643}
2644
Tom Herbert0a9627f2010-03-16 08:03:29 +00002645/*
2646 * get_rps_cpu is called from netif_receive_skb and returns the target
2647 * CPU from the RPS map of the receiving queue for a given skb.
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002648 * rcu_read_lock must be held on entry.
Tom Herbert0a9627f2010-03-16 08:03:29 +00002649 */
Tom Herbertfec5e652010-04-16 16:01:27 -07002650static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
2651 struct rps_dev_flow **rflowp)
Tom Herbert0a9627f2010-03-16 08:03:29 +00002652{
Tom Herbert0a9627f2010-03-16 08:03:29 +00002653 struct netdev_rx_queue *rxqueue;
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00002654 struct rps_map *map;
Tom Herbertfec5e652010-04-16 16:01:27 -07002655 struct rps_dev_flow_table *flow_table;
2656 struct rps_sock_flow_table *sock_flow_table;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002657 int cpu = -1;
Tom Herbertfec5e652010-04-16 16:01:27 -07002658 u16 tcpu;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002659
Tom Herbert0a9627f2010-03-16 08:03:29 +00002660 if (skb_rx_queue_recorded(skb)) {
2661 u16 index = skb_get_rx_queue(skb);
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002662 if (unlikely(index >= dev->real_num_rx_queues)) {
2663 WARN_ONCE(dev->real_num_rx_queues > 1,
2664 "%s received packet on queue %u, but number "
2665 "of RX queues is %u\n",
2666 dev->name, index, dev->real_num_rx_queues);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002667 goto done;
2668 }
2669 rxqueue = dev->_rx + index;
2670 } else
2671 rxqueue = dev->_rx;
2672
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00002673 map = rcu_dereference(rxqueue->rps_map);
2674 if (map) {
Tom Herbert85875232011-01-31 16:23:42 -08002675 if (map->len == 1 &&
2676 !rcu_dereference_raw(rxqueue->rps_flow_table)) {
Changli Gao6febfca2010-09-03 23:12:37 +00002677 tcpu = map->cpus[0];
2678 if (cpu_online(tcpu))
2679 cpu = tcpu;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002680 goto done;
Eric Dumazetb249dcb2010-04-19 21:56:38 +00002681 }
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00002682 } else if (!rcu_dereference_raw(rxqueue->rps_flow_table)) {
Tom Herbert0a9627f2010-03-16 08:03:29 +00002683 goto done;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002684 }
2685
Changli Gao2d47b452010-08-17 19:00:56 +00002686 skb_reset_network_header(skb);
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002687 if (!skb_get_rxhash(skb))
Tom Herbert0a9627f2010-03-16 08:03:29 +00002688 goto done;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002689
Tom Herbertfec5e652010-04-16 16:01:27 -07002690 flow_table = rcu_dereference(rxqueue->rps_flow_table);
2691 sock_flow_table = rcu_dereference(rps_sock_flow_table);
2692 if (flow_table && sock_flow_table) {
2693 u16 next_cpu;
2694 struct rps_dev_flow *rflow;
2695
2696 rflow = &flow_table->flows[skb->rxhash & flow_table->mask];
2697 tcpu = rflow->cpu;
2698
2699 next_cpu = sock_flow_table->ents[skb->rxhash &
2700 sock_flow_table->mask];
2701
2702 /*
2703 * If the desired CPU (where last recvmsg was done) is
2704 * different from current CPU (one in the rx-queue flow
2705 * table entry), switch if one of the following holds:
2706 * - Current CPU is unset (equal to RPS_NO_CPU).
2707 * - Current CPU is offline.
2708 * - The current CPU's queue tail has advanced beyond the
2709 * last packet that was enqueued using this table entry.
2710 * This guarantees that all previous packets for the flow
2711 * have been dequeued, thus preserving in order delivery.
2712 */
2713 if (unlikely(tcpu != next_cpu) &&
2714 (tcpu == RPS_NO_CPU || !cpu_online(tcpu) ||
2715 ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
Ben Hutchingsc4454772011-01-19 11:03:53 +00002716 rflow->last_qtail)) >= 0))
2717 rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
2718
Tom Herbertfec5e652010-04-16 16:01:27 -07002719 if (tcpu != RPS_NO_CPU && cpu_online(tcpu)) {
2720 *rflowp = rflow;
2721 cpu = tcpu;
2722 goto done;
2723 }
2724 }
2725
Tom Herbert0a9627f2010-03-16 08:03:29 +00002726 if (map) {
Tom Herbertfec5e652010-04-16 16:01:27 -07002727 tcpu = map->cpus[((u64) skb->rxhash * map->len) >> 32];
Tom Herbert0a9627f2010-03-16 08:03:29 +00002728
2729 if (cpu_online(tcpu)) {
2730 cpu = tcpu;
2731 goto done;
2732 }
2733 }
2734
2735done:
Tom Herbert0a9627f2010-03-16 08:03:29 +00002736 return cpu;
2737}
2738
Ben Hutchingsc4454772011-01-19 11:03:53 +00002739#ifdef CONFIG_RFS_ACCEL
2740
2741/**
2742 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
2743 * @dev: Device on which the filter was set
2744 * @rxq_index: RX queue index
2745 * @flow_id: Flow ID passed to ndo_rx_flow_steer()
2746 * @filter_id: Filter ID returned by ndo_rx_flow_steer()
2747 *
2748 * Drivers that implement ndo_rx_flow_steer() should periodically call
2749 * this function for each installed filter and remove the filters for
2750 * which it returns %true.
2751 */
2752bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
2753 u32 flow_id, u16 filter_id)
2754{
2755 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
2756 struct rps_dev_flow_table *flow_table;
2757 struct rps_dev_flow *rflow;
2758 bool expire = true;
2759 int cpu;
2760
2761 rcu_read_lock();
2762 flow_table = rcu_dereference(rxqueue->rps_flow_table);
2763 if (flow_table && flow_id <= flow_table->mask) {
2764 rflow = &flow_table->flows[flow_id];
2765 cpu = ACCESS_ONCE(rflow->cpu);
2766 if (rflow->filter == filter_id && cpu != RPS_NO_CPU &&
2767 ((int)(per_cpu(softnet_data, cpu).input_queue_head -
2768 rflow->last_qtail) <
2769 (int)(10 * flow_table->mask)))
2770 expire = false;
2771 }
2772 rcu_read_unlock();
2773 return expire;
2774}
2775EXPORT_SYMBOL(rps_may_expire_flow);
2776
2777#endif /* CONFIG_RFS_ACCEL */
2778
Tom Herbert0a9627f2010-03-16 08:03:29 +00002779/* Called from hardirq (IPI) context */
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002780static void rps_trigger_softirq(void *data)
Tom Herbert0a9627f2010-03-16 08:03:29 +00002781{
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002782 struct softnet_data *sd = data;
2783
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07002784 ____napi_schedule(sd, &sd->backlog);
Changli Gaodee42872010-05-02 05:42:16 +00002785 sd->received_rps++;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002786}
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002787
Tom Herbertfec5e652010-04-16 16:01:27 -07002788#endif /* CONFIG_RPS */
Tom Herbert0a9627f2010-03-16 08:03:29 +00002789
2790/*
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002791 * Check if this softnet_data structure is another cpu one
2792 * If yes, queue it to our IPI list and return 1
2793 * If no, return 0
2794 */
2795static int rps_ipi_queued(struct softnet_data *sd)
2796{
2797#ifdef CONFIG_RPS
2798 struct softnet_data *mysd = &__get_cpu_var(softnet_data);
2799
2800 if (sd != mysd) {
2801 sd->rps_ipi_next = mysd->rps_ipi_list;
2802 mysd->rps_ipi_list = sd;
2803
2804 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
2805 return 1;
2806 }
2807#endif /* CONFIG_RPS */
2808 return 0;
2809}
2810
2811/*
Tom Herbert0a9627f2010-03-16 08:03:29 +00002812 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
2813 * queue (may be a remote CPU queue).
2814 */
Tom Herbertfec5e652010-04-16 16:01:27 -07002815static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
2816 unsigned int *qtail)
Tom Herbert0a9627f2010-03-16 08:03:29 +00002817{
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002818 struct softnet_data *sd;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002819 unsigned long flags;
2820
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002821 sd = &per_cpu(softnet_data, cpu);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002822
2823 local_irq_save(flags);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002824
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002825 rps_lock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07002826 if (skb_queue_len(&sd->input_pkt_queue) <= netdev_max_backlog) {
2827 if (skb_queue_len(&sd->input_pkt_queue)) {
Tom Herbert0a9627f2010-03-16 08:03:29 +00002828enqueue:
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002829 __skb_queue_tail(&sd->input_pkt_queue, skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00002830 input_queue_tail_incr_save(sd, qtail);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002831 rps_unlock(sd);
Changli Gao152102c2010-03-30 20:16:22 +00002832 local_irq_restore(flags);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002833 return NET_RX_SUCCESS;
2834 }
2835
Eric Dumazetebda37c22010-05-06 23:51:21 +00002836 /* Schedule NAPI for backlog device
2837 * We can use non atomic operation since we own the queue lock
2838 */
2839 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002840 if (!rps_ipi_queued(sd))
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07002841 ____napi_schedule(sd, &sd->backlog);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002842 }
2843 goto enqueue;
2844 }
2845
Changli Gaodee42872010-05-02 05:42:16 +00002846 sd->dropped++;
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002847 rps_unlock(sd);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002848
Tom Herbert0a9627f2010-03-16 08:03:29 +00002849 local_irq_restore(flags);
2850
Eric Dumazetcaf586e2010-09-30 21:06:55 +00002851 atomic_long_inc(&skb->dev->rx_dropped);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002852 kfree_skb(skb);
2853 return NET_RX_DROP;
2854}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002855
Linus Torvalds1da177e2005-04-16 15:20:36 -07002856/**
2857 * netif_rx - post buffer to the network code
2858 * @skb: buffer to post
2859 *
2860 * This function receives a packet from a device driver and queues it for
2861 * the upper (protocol) levels to process. It always succeeds. The buffer
2862 * may be dropped during processing for congestion control or by the
2863 * protocol layers.
2864 *
2865 * return values:
2866 * NET_RX_SUCCESS (no congestion)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002867 * NET_RX_DROP (packet was dropped)
2868 *
2869 */
2870
2871int netif_rx(struct sk_buff *skb)
2872{
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002873 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002874
2875 /* if netpoll wants it, pretend we never saw it */
2876 if (netpoll_rx(skb))
2877 return NET_RX_DROP;
2878
Eric Dumazet3b098e22010-05-15 23:57:10 -07002879 if (netdev_tstamp_prequeue)
2880 net_timestamp_check(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002881
Koki Sanagicf66ba52010-08-23 18:45:02 +09002882 trace_netif_rx(skb);
Eric Dumazetdf334542010-03-24 19:13:54 +00002883#ifdef CONFIG_RPS
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002884 {
Tom Herbertfec5e652010-04-16 16:01:27 -07002885 struct rps_dev_flow voidflow, *rflow = &voidflow;
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002886 int cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002887
Changli Gaocece1942010-08-07 20:35:43 -07002888 preempt_disable();
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002889 rcu_read_lock();
Tom Herbertfec5e652010-04-16 16:01:27 -07002890
2891 cpu = get_rps_cpu(skb->dev, skb, &rflow);
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002892 if (cpu < 0)
2893 cpu = smp_processor_id();
Tom Herbertfec5e652010-04-16 16:01:27 -07002894
2895 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
2896
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002897 rcu_read_unlock();
Changli Gaocece1942010-08-07 20:35:43 -07002898 preempt_enable();
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002899 }
2900#else
Tom Herbertfec5e652010-04-16 16:01:27 -07002901 {
2902 unsigned int qtail;
2903 ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
2904 put_cpu();
2905 }
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002906#endif
2907 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002908}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002909EXPORT_SYMBOL(netif_rx);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002910
2911int netif_rx_ni(struct sk_buff *skb)
2912{
2913 int err;
2914
2915 preempt_disable();
2916 err = netif_rx(skb);
2917 if (local_softirq_pending())
2918 do_softirq();
2919 preempt_enable();
2920
2921 return err;
2922}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002923EXPORT_SYMBOL(netif_rx_ni);
2924
Linus Torvalds1da177e2005-04-16 15:20:36 -07002925static void net_tx_action(struct softirq_action *h)
2926{
2927 struct softnet_data *sd = &__get_cpu_var(softnet_data);
2928
2929 if (sd->completion_queue) {
2930 struct sk_buff *clist;
2931
2932 local_irq_disable();
2933 clist = sd->completion_queue;
2934 sd->completion_queue = NULL;
2935 local_irq_enable();
2936
2937 while (clist) {
2938 struct sk_buff *skb = clist;
2939 clist = clist->next;
2940
Ilpo Järvinen547b7922008-07-25 21:43:18 -07002941 WARN_ON(atomic_read(&skb->users));
Koki Sanagi07dc22e2010-08-23 18:46:12 +09002942 trace_kfree_skb(skb, net_tx_action);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002943 __kfree_skb(skb);
2944 }
2945 }
2946
2947 if (sd->output_queue) {
David S. Miller37437bb2008-07-16 02:15:04 -07002948 struct Qdisc *head;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002949
2950 local_irq_disable();
2951 head = sd->output_queue;
2952 sd->output_queue = NULL;
Changli Gaoa9cbd582010-04-26 23:06:24 +00002953 sd->output_queue_tailp = &sd->output_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002954 local_irq_enable();
2955
2956 while (head) {
David S. Miller37437bb2008-07-16 02:15:04 -07002957 struct Qdisc *q = head;
2958 spinlock_t *root_lock;
2959
Linus Torvalds1da177e2005-04-16 15:20:36 -07002960 head = head->next_sched;
2961
David S. Miller5fb66222008-08-02 20:02:43 -07002962 root_lock = qdisc_lock(q);
David S. Miller37437bb2008-07-16 02:15:04 -07002963 if (spin_trylock(root_lock)) {
Jarek Poplawskidef82a12008-08-17 21:54:43 -07002964 smp_mb__before_clear_bit();
2965 clear_bit(__QDISC_STATE_SCHED,
2966 &q->state);
David S. Miller37437bb2008-07-16 02:15:04 -07002967 qdisc_run(q);
2968 spin_unlock(root_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002969 } else {
David S. Miller195648b2008-08-19 04:00:36 -07002970 if (!test_bit(__QDISC_STATE_DEACTIVATED,
Jarek Poplawskie8a83e12008-09-07 18:41:21 -07002971 &q->state)) {
David S. Miller195648b2008-08-19 04:00:36 -07002972 __netif_reschedule(q);
Jarek Poplawskie8a83e12008-09-07 18:41:21 -07002973 } else {
2974 smp_mb__before_clear_bit();
2975 clear_bit(__QDISC_STATE_SCHED,
2976 &q->state);
2977 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002978 }
2979 }
2980 }
2981}
2982
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00002983#if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \
2984 (defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE))
Michał Mirosławda678292009-06-05 05:35:28 +00002985/* This hook is defined here for ATM LANE */
2986int (*br_fdb_test_addr_hook)(struct net_device *dev,
2987 unsigned char *addr) __read_mostly;
Stephen Hemminger4fb019a2009-09-11 11:50:08 -07002988EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
Michał Mirosławda678292009-06-05 05:35:28 +00002989#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002990
Linus Torvalds1da177e2005-04-16 15:20:36 -07002991#ifdef CONFIG_NET_CLS_ACT
2992/* TODO: Maybe we should just force sch_ingress to be compiled in
2993 * when CONFIG_NET_CLS_ACT is? otherwise some useless instructions
2994 * a compare and 2 stores extra right now if we dont have it on
2995 * but have CONFIG_NET_CLS_ACT
Lucas De Marchi25985ed2011-03-30 22:57:33 -03002996 * NOTE: This doesn't stop any functionality; if you dont have
2997 * the ingress scheduler, you just can't add policies on ingress.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002998 *
2999 */
Eric Dumazet24824a02010-10-02 06:11:55 +00003000static int ing_filter(struct sk_buff *skb, struct netdev_queue *rxq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003001{
Linus Torvalds1da177e2005-04-16 15:20:36 -07003002 struct net_device *dev = skb->dev;
Herbert Xuf697c3e2007-10-14 00:38:47 -07003003 u32 ttl = G_TC_RTTL(skb->tc_verd);
David S. Miller555353c2008-07-08 17:33:13 -07003004 int result = TC_ACT_OK;
3005 struct Qdisc *q;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003006
Stephen Hemmingerde384832010-08-01 00:33:23 -07003007 if (unlikely(MAX_RED_LOOP < ttl++)) {
3008 if (net_ratelimit())
3009 pr_warning( "Redir loop detected Dropping packet (%d->%d)\n",
3010 skb->skb_iif, dev->ifindex);
Herbert Xuf697c3e2007-10-14 00:38:47 -07003011 return TC_ACT_SHOT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003012 }
3013
Herbert Xuf697c3e2007-10-14 00:38:47 -07003014 skb->tc_verd = SET_TC_RTTL(skb->tc_verd, ttl);
3015 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS);
3016
David S. Miller83874002008-07-17 00:53:03 -07003017 q = rxq->qdisc;
David S. Miller8d50b532008-07-30 02:37:46 -07003018 if (q != &noop_qdisc) {
David S. Miller83874002008-07-17 00:53:03 -07003019 spin_lock(qdisc_lock(q));
David S. Millera9312ae2008-08-17 21:51:03 -07003020 if (likely(!test_bit(__QDISC_STATE_DEACTIVATED, &q->state)))
3021 result = qdisc_enqueue_root(skb, q);
David S. Miller83874002008-07-17 00:53:03 -07003022 spin_unlock(qdisc_lock(q));
3023 }
Herbert Xuf697c3e2007-10-14 00:38:47 -07003024
Linus Torvalds1da177e2005-04-16 15:20:36 -07003025 return result;
3026}
Herbert Xuf697c3e2007-10-14 00:38:47 -07003027
3028static inline struct sk_buff *handle_ing(struct sk_buff *skb,
3029 struct packet_type **pt_prev,
3030 int *ret, struct net_device *orig_dev)
3031{
Eric Dumazet24824a02010-10-02 06:11:55 +00003032 struct netdev_queue *rxq = rcu_dereference(skb->dev->ingress_queue);
3033
3034 if (!rxq || rxq->qdisc == &noop_qdisc)
Herbert Xuf697c3e2007-10-14 00:38:47 -07003035 goto out;
3036
3037 if (*pt_prev) {
3038 *ret = deliver_skb(skb, *pt_prev, orig_dev);
3039 *pt_prev = NULL;
Herbert Xuf697c3e2007-10-14 00:38:47 -07003040 }
3041
Eric Dumazet24824a02010-10-02 06:11:55 +00003042 switch (ing_filter(skb, rxq)) {
Herbert Xuf697c3e2007-10-14 00:38:47 -07003043 case TC_ACT_SHOT:
3044 case TC_ACT_STOLEN:
3045 kfree_skb(skb);
3046 return NULL;
3047 }
3048
3049out:
3050 skb->tc_verd = 0;
3051 return skb;
3052}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003053#endif
3054
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003055/**
3056 * netdev_rx_handler_register - register receive handler
3057 * @dev: device to register a handler for
3058 * @rx_handler: receive handler to register
Jiri Pirko93e2c322010-06-10 03:34:59 +00003059 * @rx_handler_data: data pointer that is used by rx handler
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003060 *
3061 * Register a receive hander for a device. This handler will then be
3062 * called from __netif_receive_skb. A negative errno code is returned
3063 * on a failure.
3064 *
3065 * The caller must hold the rtnl_mutex.
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003066 *
3067 * For a general description of rx_handler, see enum rx_handler_result.
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003068 */
3069int netdev_rx_handler_register(struct net_device *dev,
Jiri Pirko93e2c322010-06-10 03:34:59 +00003070 rx_handler_func_t *rx_handler,
3071 void *rx_handler_data)
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003072{
3073 ASSERT_RTNL();
3074
3075 if (dev->rx_handler)
3076 return -EBUSY;
3077
Jiri Pirko93e2c322010-06-10 03:34:59 +00003078 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003079 rcu_assign_pointer(dev->rx_handler, rx_handler);
3080
3081 return 0;
3082}
3083EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
3084
3085/**
3086 * netdev_rx_handler_unregister - unregister receive handler
3087 * @dev: device to unregister a handler from
3088 *
3089 * Unregister a receive hander from a device.
3090 *
3091 * The caller must hold the rtnl_mutex.
3092 */
3093void netdev_rx_handler_unregister(struct net_device *dev)
3094{
3095
3096 ASSERT_RTNL();
3097 rcu_assign_pointer(dev->rx_handler, NULL);
Jiri Pirko93e2c322010-06-10 03:34:59 +00003098 rcu_assign_pointer(dev->rx_handler_data, NULL);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003099}
3100EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
3101
Eric Dumazet10f744d2010-03-28 23:07:20 -07003102static int __netif_receive_skb(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003103{
3104 struct packet_type *ptype, *pt_prev;
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003105 rx_handler_func_t *rx_handler;
David S. Millerf2ccd8f2005-08-09 19:34:12 -07003106 struct net_device *orig_dev;
David S. Miller63d8ea72011-02-28 10:48:59 -08003107 struct net_device *null_or_dev;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003108 bool deliver_exact = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003109 int ret = NET_RX_DROP;
Al Viro252e3342006-11-14 20:48:11 -08003110 __be16 type;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003111
Eric Dumazet3b098e22010-05-15 23:57:10 -07003112 if (!netdev_tstamp_prequeue)
3113 net_timestamp_check(skb);
Eric Dumazet81bbb3d2009-09-30 16:42:42 -07003114
Koki Sanagicf66ba52010-08-23 18:45:02 +09003115 trace_netif_receive_skb(skb);
Patrick McHardy9b22ea52008-11-04 14:49:57 -08003116
Linus Torvalds1da177e2005-04-16 15:20:36 -07003117 /* if we've gotten here through NAPI, check netpoll */
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003118 if (netpoll_receive_skb(skb))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003119 return NET_RX_DROP;
3120
Eric Dumazet8964be42009-11-20 15:35:04 -08003121 if (!skb->skb_iif)
3122 skb->skb_iif = skb->dev->ifindex;
Joe Eykholtcc9bd5c2008-07-02 18:22:00 -07003123 orig_dev = skb->dev;
Jiri Pirko1765a572011-02-12 06:48:36 +00003124
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -07003125 skb_reset_network_header(skb);
Arnaldo Carvalho de Melobadff6d2007-03-13 13:06:52 -03003126 skb_reset_transport_header(skb);
Jiri Pirko0b5c9db2011-06-10 06:56:58 +00003127 skb_reset_mac_len(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003128
3129 pt_prev = NULL;
3130
3131 rcu_read_lock();
3132
David S. Miller63d8ea72011-02-28 10:48:59 -08003133another_round:
3134
3135 __this_cpu_inc(softnet_data.processed);
3136
Jiri Pirkobcc6d472011-04-07 19:48:33 +00003137 if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
3138 skb = vlan_untag(skb);
3139 if (unlikely(!skb))
3140 goto out;
3141 }
3142
Linus Torvalds1da177e2005-04-16 15:20:36 -07003143#ifdef CONFIG_NET_CLS_ACT
3144 if (skb->tc_verd & TC_NCLS) {
3145 skb->tc_verd = CLR_TC_NCLS(skb->tc_verd);
3146 goto ncls;
3147 }
3148#endif
3149
3150 list_for_each_entry_rcu(ptype, &ptype_all, list) {
David S. Miller63d8ea72011-02-28 10:48:59 -08003151 if (!ptype->dev || ptype->dev == skb->dev) {
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003152 if (pt_prev)
David S. Millerf2ccd8f2005-08-09 19:34:12 -07003153 ret = deliver_skb(skb, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003154 pt_prev = ptype;
3155 }
3156 }
3157
3158#ifdef CONFIG_NET_CLS_ACT
Herbert Xuf697c3e2007-10-14 00:38:47 -07003159 skb = handle_ing(skb, &pt_prev, &ret, orig_dev);
3160 if (!skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003161 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003162ncls:
3163#endif
3164
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003165 rx_handler = rcu_dereference(skb->dev->rx_handler);
3166 if (rx_handler) {
3167 if (pt_prev) {
3168 ret = deliver_skb(skb, pt_prev, orig_dev);
3169 pt_prev = NULL;
3170 }
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003171 switch (rx_handler(&skb)) {
3172 case RX_HANDLER_CONSUMED:
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003173 goto out;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003174 case RX_HANDLER_ANOTHER:
David S. Miller63d8ea72011-02-28 10:48:59 -08003175 goto another_round;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003176 case RX_HANDLER_EXACT:
3177 deliver_exact = true;
3178 case RX_HANDLER_PASS:
3179 break;
3180 default:
3181 BUG();
3182 }
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003183 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003184
Jesse Gross3701e512010-10-20 13:56:06 +00003185 if (vlan_tx_tag_present(skb)) {
3186 if (pt_prev) {
3187 ret = deliver_skb(skb, pt_prev, orig_dev);
3188 pt_prev = NULL;
3189 }
Jiri Pirkobcc6d472011-04-07 19:48:33 +00003190 if (vlan_do_receive(&skb)) {
Jesse Gross3701e512010-10-20 13:56:06 +00003191 ret = __netif_receive_skb(skb);
3192 goto out;
3193 } else if (unlikely(!skb))
3194 goto out;
3195 }
3196
David S. Miller63d8ea72011-02-28 10:48:59 -08003197 /* deliver only exact match when indicated */
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003198 null_or_dev = deliver_exact ? skb->dev : NULL;
Andy Gospodarek1f3c8802009-12-14 10:48:58 +00003199
Linus Torvalds1da177e2005-04-16 15:20:36 -07003200 type = skb->protocol;
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08003201 list_for_each_entry_rcu(ptype,
3202 &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
David S. Miller63d8ea72011-02-28 10:48:59 -08003203 if (ptype->type == type &&
Jiri Pirkoe3f48d32011-02-28 20:26:31 +00003204 (ptype->dev == null_or_dev || ptype->dev == skb->dev ||
3205 ptype->dev == orig_dev)) {
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003206 if (pt_prev)
David S. Millerf2ccd8f2005-08-09 19:34:12 -07003207 ret = deliver_skb(skb, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003208 pt_prev = ptype;
3209 }
3210 }
3211
3212 if (pt_prev) {
David S. Millerf2ccd8f2005-08-09 19:34:12 -07003213 ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003214 } else {
Eric Dumazetcaf586e2010-09-30 21:06:55 +00003215 atomic_long_inc(&skb->dev->rx_dropped);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003216 kfree_skb(skb);
3217 /* Jamal, now you will not able to escape explaining
3218 * me how you were going to use this. :-)
3219 */
3220 ret = NET_RX_DROP;
3221 }
3222
3223out:
3224 rcu_read_unlock();
3225 return ret;
3226}
Tom Herbert0a9627f2010-03-16 08:03:29 +00003227
3228/**
3229 * netif_receive_skb - process receive buffer from network
3230 * @skb: buffer to process
3231 *
3232 * netif_receive_skb() is the main receive data processing function.
3233 * It always succeeds. The buffer may be dropped during processing
3234 * for congestion control or by the protocol layers.
3235 *
3236 * This function may only be called from softirq context and interrupts
3237 * should be enabled.
3238 *
3239 * Return values (usually ignored):
3240 * NET_RX_SUCCESS: no congestion
3241 * NET_RX_DROP: packet was dropped
3242 */
3243int netif_receive_skb(struct sk_buff *skb)
3244{
Eric Dumazet3b098e22010-05-15 23:57:10 -07003245 if (netdev_tstamp_prequeue)
3246 net_timestamp_check(skb);
3247
Richard Cochranc1f19b52010-07-17 08:49:36 +00003248 if (skb_defer_rx_timestamp(skb))
3249 return NET_RX_SUCCESS;
3250
Eric Dumazetdf334542010-03-24 19:13:54 +00003251#ifdef CONFIG_RPS
Eric Dumazet3b098e22010-05-15 23:57:10 -07003252 {
3253 struct rps_dev_flow voidflow, *rflow = &voidflow;
3254 int cpu, ret;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003255
Eric Dumazet3b098e22010-05-15 23:57:10 -07003256 rcu_read_lock();
Tom Herbert0a9627f2010-03-16 08:03:29 +00003257
Eric Dumazet3b098e22010-05-15 23:57:10 -07003258 cpu = get_rps_cpu(skb->dev, skb, &rflow);
Tom Herbertfec5e652010-04-16 16:01:27 -07003259
Eric Dumazet3b098e22010-05-15 23:57:10 -07003260 if (cpu >= 0) {
3261 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3262 rcu_read_unlock();
3263 } else {
3264 rcu_read_unlock();
3265 ret = __netif_receive_skb(skb);
3266 }
3267
3268 return ret;
Tom Herbertfec5e652010-04-16 16:01:27 -07003269 }
Tom Herbert1e94d722010-03-18 17:45:44 -07003270#else
3271 return __netif_receive_skb(skb);
3272#endif
Tom Herbert0a9627f2010-03-16 08:03:29 +00003273}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003274EXPORT_SYMBOL(netif_receive_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003275
Eric Dumazet88751272010-04-19 05:07:33 +00003276/* Network device is going away, flush any packets still pending
3277 * Called with irqs disabled.
3278 */
Changli Gao152102c2010-03-30 20:16:22 +00003279static void flush_backlog(void *arg)
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003280{
Changli Gao152102c2010-03-30 20:16:22 +00003281 struct net_device *dev = arg;
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003282 struct softnet_data *sd = &__get_cpu_var(softnet_data);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003283 struct sk_buff *skb, *tmp;
3284
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003285 rps_lock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07003286 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003287 if (skb->dev == dev) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003288 __skb_unlink(skb, &sd->input_pkt_queue);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003289 kfree_skb(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003290 input_queue_head_incr(sd);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003291 }
Changli Gao6e7676c2010-04-27 15:07:33 -07003292 }
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003293 rps_unlock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07003294
3295 skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
3296 if (skb->dev == dev) {
3297 __skb_unlink(skb, &sd->process_queue);
3298 kfree_skb(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003299 input_queue_head_incr(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07003300 }
3301 }
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003302}
3303
Herbert Xud565b0a2008-12-15 23:38:52 -08003304static int napi_gro_complete(struct sk_buff *skb)
3305{
3306 struct packet_type *ptype;
3307 __be16 type = skb->protocol;
3308 struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK];
3309 int err = -ENOENT;
3310
Herbert Xufc59f9a2009-04-14 15:11:06 -07003311 if (NAPI_GRO_CB(skb)->count == 1) {
3312 skb_shinfo(skb)->gso_size = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003313 goto out;
Herbert Xufc59f9a2009-04-14 15:11:06 -07003314 }
Herbert Xud565b0a2008-12-15 23:38:52 -08003315
3316 rcu_read_lock();
3317 list_for_each_entry_rcu(ptype, head, list) {
3318 if (ptype->type != type || ptype->dev || !ptype->gro_complete)
3319 continue;
3320
3321 err = ptype->gro_complete(skb);
3322 break;
3323 }
3324 rcu_read_unlock();
3325
3326 if (err) {
3327 WARN_ON(&ptype->list == head);
3328 kfree_skb(skb);
3329 return NET_RX_SUCCESS;
3330 }
3331
3332out:
Herbert Xud565b0a2008-12-15 23:38:52 -08003333 return netif_receive_skb(skb);
3334}
3335
Eric Dumazet86cac582010-08-31 18:25:32 +00003336inline void napi_gro_flush(struct napi_struct *napi)
Herbert Xud565b0a2008-12-15 23:38:52 -08003337{
3338 struct sk_buff *skb, *next;
3339
3340 for (skb = napi->gro_list; skb; skb = next) {
3341 next = skb->next;
3342 skb->next = NULL;
3343 napi_gro_complete(skb);
3344 }
3345
Herbert Xu4ae55442009-02-08 18:00:36 +00003346 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003347 napi->gro_list = NULL;
3348}
Eric Dumazet86cac582010-08-31 18:25:32 +00003349EXPORT_SYMBOL(napi_gro_flush);
Herbert Xud565b0a2008-12-15 23:38:52 -08003350
Ben Hutchings5b252f02009-10-29 07:17:09 +00003351enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xud565b0a2008-12-15 23:38:52 -08003352{
3353 struct sk_buff **pp = NULL;
3354 struct packet_type *ptype;
3355 __be16 type = skb->protocol;
3356 struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK];
Herbert Xu0da2afd52008-12-26 14:57:42 -08003357 int same_flow;
Herbert Xud565b0a2008-12-15 23:38:52 -08003358 int mac_len;
Ben Hutchings5b252f02009-10-29 07:17:09 +00003359 enum gro_result ret;
Herbert Xud565b0a2008-12-15 23:38:52 -08003360
Jarek Poplawskice9e76c2010-08-05 01:19:11 +00003361 if (!(skb->dev->features & NETIF_F_GRO) || netpoll_rx_on(skb))
Herbert Xud565b0a2008-12-15 23:38:52 -08003362 goto normal;
3363
David S. Miller21dc3302010-08-23 00:13:46 -07003364 if (skb_is_gso(skb) || skb_has_frag_list(skb))
Herbert Xuf17f5c92009-01-14 14:36:12 -08003365 goto normal;
3366
Herbert Xud565b0a2008-12-15 23:38:52 -08003367 rcu_read_lock();
3368 list_for_each_entry_rcu(ptype, head, list) {
Herbert Xud565b0a2008-12-15 23:38:52 -08003369 if (ptype->type != type || ptype->dev || !ptype->gro_receive)
3370 continue;
3371
Herbert Xu86911732009-01-29 14:19:50 +00003372 skb_set_network_header(skb, skb_gro_offset(skb));
Herbert Xud565b0a2008-12-15 23:38:52 -08003373 mac_len = skb->network_header - skb->mac_header;
3374 skb->mac_len = mac_len;
3375 NAPI_GRO_CB(skb)->same_flow = 0;
3376 NAPI_GRO_CB(skb)->flush = 0;
Herbert Xu5d38a072009-01-04 16:13:40 -08003377 NAPI_GRO_CB(skb)->free = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003378
Herbert Xud565b0a2008-12-15 23:38:52 -08003379 pp = ptype->gro_receive(&napi->gro_list, skb);
3380 break;
3381 }
3382 rcu_read_unlock();
3383
3384 if (&ptype->list == head)
3385 goto normal;
3386
Herbert Xu0da2afd52008-12-26 14:57:42 -08003387 same_flow = NAPI_GRO_CB(skb)->same_flow;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003388 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
Herbert Xu0da2afd52008-12-26 14:57:42 -08003389
Herbert Xud565b0a2008-12-15 23:38:52 -08003390 if (pp) {
3391 struct sk_buff *nskb = *pp;
3392
3393 *pp = nskb->next;
3394 nskb->next = NULL;
3395 napi_gro_complete(nskb);
Herbert Xu4ae55442009-02-08 18:00:36 +00003396 napi->gro_count--;
Herbert Xud565b0a2008-12-15 23:38:52 -08003397 }
3398
Herbert Xu0da2afd52008-12-26 14:57:42 -08003399 if (same_flow)
Herbert Xud565b0a2008-12-15 23:38:52 -08003400 goto ok;
3401
Herbert Xu4ae55442009-02-08 18:00:36 +00003402 if (NAPI_GRO_CB(skb)->flush || napi->gro_count >= MAX_GRO_SKBS)
Herbert Xud565b0a2008-12-15 23:38:52 -08003403 goto normal;
Herbert Xud565b0a2008-12-15 23:38:52 -08003404
Herbert Xu4ae55442009-02-08 18:00:36 +00003405 napi->gro_count++;
Herbert Xud565b0a2008-12-15 23:38:52 -08003406 NAPI_GRO_CB(skb)->count = 1;
Herbert Xu86911732009-01-29 14:19:50 +00003407 skb_shinfo(skb)->gso_size = skb_gro_len(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08003408 skb->next = napi->gro_list;
3409 napi->gro_list = skb;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003410 ret = GRO_HELD;
Herbert Xud565b0a2008-12-15 23:38:52 -08003411
Herbert Xuad0f9902009-02-01 01:24:55 -08003412pull:
Herbert Xucb189782009-05-26 18:50:31 +00003413 if (skb_headlen(skb) < skb_gro_offset(skb)) {
3414 int grow = skb_gro_offset(skb) - skb_headlen(skb);
3415
3416 BUG_ON(skb->end - skb->tail < grow);
3417
3418 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
3419
3420 skb->tail += grow;
3421 skb->data_len -= grow;
3422
3423 skb_shinfo(skb)->frags[0].page_offset += grow;
3424 skb_shinfo(skb)->frags[0].size -= grow;
3425
3426 if (unlikely(!skb_shinfo(skb)->frags[0].size)) {
3427 put_page(skb_shinfo(skb)->frags[0].page);
3428 memmove(skb_shinfo(skb)->frags,
3429 skb_shinfo(skb)->frags + 1,
Jarek Poplawskie5093ae2010-08-11 02:02:10 +00003430 --skb_shinfo(skb)->nr_frags * sizeof(skb_frag_t));
Herbert Xucb189782009-05-26 18:50:31 +00003431 }
Herbert Xuad0f9902009-02-01 01:24:55 -08003432 }
3433
Herbert Xud565b0a2008-12-15 23:38:52 -08003434ok:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003435 return ret;
Herbert Xud565b0a2008-12-15 23:38:52 -08003436
3437normal:
Herbert Xuad0f9902009-02-01 01:24:55 -08003438 ret = GRO_NORMAL;
3439 goto pull;
Herbert Xu5d38a072009-01-04 16:13:40 -08003440}
Herbert Xu96e93ea2009-01-06 10:49:34 -08003441EXPORT_SYMBOL(dev_gro_receive);
3442
Eric Dumazet40d08022010-08-26 22:03:08 -07003443static inline gro_result_t
Ben Hutchings5b252f02009-10-29 07:17:09 +00003444__napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu96e93ea2009-01-06 10:49:34 -08003445{
3446 struct sk_buff *p;
3447
3448 for (p = napi->gro_list; p; p = p->next) {
Eric Dumazet40d08022010-08-26 22:03:08 -07003449 unsigned long diffs;
3450
3451 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
Jesse Gross3701e512010-10-20 13:56:06 +00003452 diffs |= p->vlan_tci ^ skb->vlan_tci;
Eric Dumazet40d08022010-08-26 22:03:08 -07003453 diffs |= compare_ether_header(skb_mac_header(p),
Joe Perchesf64f9e72009-11-29 16:55:45 -08003454 skb_gro_mac_header(skb));
Eric Dumazet40d08022010-08-26 22:03:08 -07003455 NAPI_GRO_CB(p)->same_flow = !diffs;
Herbert Xu96e93ea2009-01-06 10:49:34 -08003456 NAPI_GRO_CB(p)->flush = 0;
3457 }
3458
3459 return dev_gro_receive(napi, skb);
3460}
Herbert Xu5d38a072009-01-04 16:13:40 -08003461
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003462gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
Herbert Xu5d38a072009-01-04 16:13:40 -08003463{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003464 switch (ret) {
3465 case GRO_NORMAL:
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003466 if (netif_receive_skb(skb))
3467 ret = GRO_DROP;
3468 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08003469
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003470 case GRO_DROP:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003471 case GRO_MERGED_FREE:
Herbert Xu5d38a072009-01-04 16:13:40 -08003472 kfree_skb(skb);
3473 break;
Ben Hutchings5b252f02009-10-29 07:17:09 +00003474
3475 case GRO_HELD:
3476 case GRO_MERGED:
3477 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08003478 }
3479
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003480 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003481}
3482EXPORT_SYMBOL(napi_skb_finish);
3483
Herbert Xu78a478d2009-05-26 18:50:21 +00003484void skb_gro_reset_offset(struct sk_buff *skb)
3485{
3486 NAPI_GRO_CB(skb)->data_offset = 0;
3487 NAPI_GRO_CB(skb)->frag0 = NULL;
Herbert Xu74895942009-05-26 18:50:27 +00003488 NAPI_GRO_CB(skb)->frag0_len = 0;
Herbert Xu78a478d2009-05-26 18:50:21 +00003489
Herbert Xu78d3fd02009-05-26 18:50:23 +00003490 if (skb->mac_header == skb->tail &&
Herbert Xu74895942009-05-26 18:50:27 +00003491 !PageHighMem(skb_shinfo(skb)->frags[0].page)) {
Herbert Xu78a478d2009-05-26 18:50:21 +00003492 NAPI_GRO_CB(skb)->frag0 =
3493 page_address(skb_shinfo(skb)->frags[0].page) +
3494 skb_shinfo(skb)->frags[0].page_offset;
Herbert Xu74895942009-05-26 18:50:27 +00003495 NAPI_GRO_CB(skb)->frag0_len = skb_shinfo(skb)->frags[0].size;
3496 }
Herbert Xu78a478d2009-05-26 18:50:21 +00003497}
3498EXPORT_SYMBOL(skb_gro_reset_offset);
3499
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003500gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003501{
Herbert Xu86911732009-01-29 14:19:50 +00003502 skb_gro_reset_offset(skb);
3503
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003504 return napi_skb_finish(__napi_gro_receive(napi, skb), skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08003505}
3506EXPORT_SYMBOL(napi_gro_receive);
3507
stephen hemmingerd0c2b0d2010-10-19 07:12:10 +00003508static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu96e93ea2009-01-06 10:49:34 -08003509{
Herbert Xu96e93ea2009-01-06 10:49:34 -08003510 __skb_pull(skb, skb_headlen(skb));
3511 skb_reserve(skb, NET_IP_ALIGN - skb_headroom(skb));
Jesse Gross3701e512010-10-20 13:56:06 +00003512 skb->vlan_tci = 0;
Herbert Xu66c46d72011-01-29 20:44:54 -08003513 skb->dev = napi->dev;
Andy Gospodarek6d152e22011-02-02 14:53:25 -08003514 skb->skb_iif = 0;
Herbert Xu96e93ea2009-01-06 10:49:34 -08003515
3516 napi->skb = skb;
3517}
Herbert Xu96e93ea2009-01-06 10:49:34 -08003518
Herbert Xu76620aa2009-04-16 02:02:07 -07003519struct sk_buff *napi_get_frags(struct napi_struct *napi)
Herbert Xu5d38a072009-01-04 16:13:40 -08003520{
Herbert Xu5d38a072009-01-04 16:13:40 -08003521 struct sk_buff *skb = napi->skb;
Herbert Xu5d38a072009-01-04 16:13:40 -08003522
3523 if (!skb) {
Eric Dumazet89d71a62009-10-13 05:34:20 +00003524 skb = netdev_alloc_skb_ip_align(napi->dev, GRO_MAX_HEAD);
3525 if (skb)
3526 napi->skb = skb;
Herbert Xu5d38a072009-01-04 16:13:40 -08003527 }
Herbert Xu96e93ea2009-01-06 10:49:34 -08003528 return skb;
3529}
Herbert Xu76620aa2009-04-16 02:02:07 -07003530EXPORT_SYMBOL(napi_get_frags);
Herbert Xu96e93ea2009-01-06 10:49:34 -08003531
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003532gro_result_t napi_frags_finish(struct napi_struct *napi, struct sk_buff *skb,
3533 gro_result_t ret)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003534{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003535 switch (ret) {
3536 case GRO_NORMAL:
Herbert Xu86911732009-01-29 14:19:50 +00003537 case GRO_HELD:
Ajit Khapardee76b69c2010-02-16 20:25:43 +00003538 skb->protocol = eth_type_trans(skb, skb->dev);
Herbert Xu86911732009-01-29 14:19:50 +00003539
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003540 if (ret == GRO_HELD)
3541 skb_gro_pull(skb, -ETH_HLEN);
3542 else if (netif_receive_skb(skb))
3543 ret = GRO_DROP;
Herbert Xu86911732009-01-29 14:19:50 +00003544 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003545
3546 case GRO_DROP:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003547 case GRO_MERGED_FREE:
3548 napi_reuse_skb(napi, skb);
3549 break;
Ben Hutchings5b252f02009-10-29 07:17:09 +00003550
3551 case GRO_MERGED:
3552 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003553 }
3554
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003555 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003556}
3557EXPORT_SYMBOL(napi_frags_finish);
3558
Herbert Xu76620aa2009-04-16 02:02:07 -07003559struct sk_buff *napi_frags_skb(struct napi_struct *napi)
Herbert Xu96e93ea2009-01-06 10:49:34 -08003560{
Herbert Xu76620aa2009-04-16 02:02:07 -07003561 struct sk_buff *skb = napi->skb;
3562 struct ethhdr *eth;
Herbert Xua5b1cf22009-05-26 18:50:28 +00003563 unsigned int hlen;
3564 unsigned int off;
Herbert Xu76620aa2009-04-16 02:02:07 -07003565
3566 napi->skb = NULL;
3567
3568 skb_reset_mac_header(skb);
3569 skb_gro_reset_offset(skb);
3570
Herbert Xua5b1cf22009-05-26 18:50:28 +00003571 off = skb_gro_offset(skb);
3572 hlen = off + sizeof(*eth);
3573 eth = skb_gro_header_fast(skb, off);
3574 if (skb_gro_header_hard(skb, hlen)) {
3575 eth = skb_gro_header_slow(skb, hlen, off);
3576 if (unlikely(!eth)) {
3577 napi_reuse_skb(napi, skb);
3578 skb = NULL;
3579 goto out;
3580 }
Herbert Xu76620aa2009-04-16 02:02:07 -07003581 }
3582
3583 skb_gro_pull(skb, sizeof(*eth));
3584
3585 /*
3586 * This works because the only protocols we care about don't require
3587 * special handling. We'll fix it up properly at the end.
3588 */
3589 skb->protocol = eth->h_proto;
3590
3591out:
3592 return skb;
3593}
3594EXPORT_SYMBOL(napi_frags_skb);
3595
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003596gro_result_t napi_gro_frags(struct napi_struct *napi)
Herbert Xu76620aa2009-04-16 02:02:07 -07003597{
3598 struct sk_buff *skb = napi_frags_skb(napi);
Herbert Xu96e93ea2009-01-06 10:49:34 -08003599
3600 if (!skb)
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003601 return GRO_DROP;
Herbert Xu96e93ea2009-01-06 10:49:34 -08003602
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003603 return napi_frags_finish(napi, skb, __napi_gro_receive(napi, skb));
Herbert Xu5d38a072009-01-04 16:13:40 -08003604}
3605EXPORT_SYMBOL(napi_gro_frags);
3606
Eric Dumazete326bed2010-04-22 00:22:45 -07003607/*
3608 * net_rps_action sends any pending IPI's for rps.
3609 * Note: called with local irq disabled, but exits with local irq enabled.
3610 */
3611static void net_rps_action_and_irq_enable(struct softnet_data *sd)
3612{
3613#ifdef CONFIG_RPS
3614 struct softnet_data *remsd = sd->rps_ipi_list;
3615
3616 if (remsd) {
3617 sd->rps_ipi_list = NULL;
3618
3619 local_irq_enable();
3620
3621 /* Send pending IPI's to kick RPS processing on remote cpus. */
3622 while (remsd) {
3623 struct softnet_data *next = remsd->rps_ipi_next;
3624
3625 if (cpu_online(remsd->cpu))
3626 __smp_call_function_single(remsd->cpu,
3627 &remsd->csd, 0);
3628 remsd = next;
3629 }
3630 } else
3631#endif
3632 local_irq_enable();
3633}
3634
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003635static int process_backlog(struct napi_struct *napi, int quota)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003636{
3637 int work = 0;
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003638 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003639
Eric Dumazete326bed2010-04-22 00:22:45 -07003640#ifdef CONFIG_RPS
3641 /* Check if we have pending ipi, its better to send them now,
3642 * not waiting net_rx_action() end.
3643 */
3644 if (sd->rps_ipi_list) {
3645 local_irq_disable();
3646 net_rps_action_and_irq_enable(sd);
3647 }
3648#endif
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003649 napi->weight = weight_p;
Changli Gao6e7676c2010-04-27 15:07:33 -07003650 local_irq_disable();
3651 while (work < quota) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003652 struct sk_buff *skb;
Changli Gao6e7676c2010-04-27 15:07:33 -07003653 unsigned int qlen;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003654
Changli Gao6e7676c2010-04-27 15:07:33 -07003655 while ((skb = __skb_dequeue(&sd->process_queue))) {
Eric Dumazete4008272010-04-05 15:42:39 -07003656 local_irq_enable();
Changli Gao6e7676c2010-04-27 15:07:33 -07003657 __netif_receive_skb(skb);
Changli Gao6e7676c2010-04-27 15:07:33 -07003658 local_irq_disable();
Tom Herbert76cc8b12010-05-20 18:37:59 +00003659 input_queue_head_incr(sd);
3660 if (++work >= quota) {
3661 local_irq_enable();
3662 return work;
3663 }
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003664 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003665
Changli Gao6e7676c2010-04-27 15:07:33 -07003666 rps_lock(sd);
3667 qlen = skb_queue_len(&sd->input_pkt_queue);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003668 if (qlen)
Changli Gao6e7676c2010-04-27 15:07:33 -07003669 skb_queue_splice_tail_init(&sd->input_pkt_queue,
3670 &sd->process_queue);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003671
Changli Gao6e7676c2010-04-27 15:07:33 -07003672 if (qlen < quota - work) {
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003673 /*
3674 * Inline a custom version of __napi_complete().
3675 * only current cpu owns and manipulates this napi,
3676 * and NAPI_STATE_SCHED is the only possible flag set on backlog.
3677 * we can use a plain write instead of clear_bit(),
3678 * and we dont need an smp_mb() memory barrier.
3679 */
3680 list_del(&napi->poll_list);
3681 napi->state = 0;
3682
Changli Gao6e7676c2010-04-27 15:07:33 -07003683 quota = work + qlen;
3684 }
3685 rps_unlock(sd);
3686 }
3687 local_irq_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003688
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003689 return work;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003690}
3691
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003692/**
3693 * __napi_schedule - schedule for receive
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07003694 * @n: entry to schedule
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003695 *
3696 * The entry's receive function will be scheduled to run
3697 */
Harvey Harrisonb5606c22008-02-13 15:03:16 -08003698void __napi_schedule(struct napi_struct *n)
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003699{
3700 unsigned long flags;
3701
3702 local_irq_save(flags);
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003703 ____napi_schedule(&__get_cpu_var(softnet_data), n);
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003704 local_irq_restore(flags);
3705}
3706EXPORT_SYMBOL(__napi_schedule);
3707
Herbert Xud565b0a2008-12-15 23:38:52 -08003708void __napi_complete(struct napi_struct *n)
3709{
3710 BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
3711 BUG_ON(n->gro_list);
3712
3713 list_del(&n->poll_list);
3714 smp_mb__before_clear_bit();
3715 clear_bit(NAPI_STATE_SCHED, &n->state);
3716}
3717EXPORT_SYMBOL(__napi_complete);
3718
3719void napi_complete(struct napi_struct *n)
3720{
3721 unsigned long flags;
3722
3723 /*
3724 * don't let napi dequeue from the cpu poll list
3725 * just in case its running on a different cpu
3726 */
3727 if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state)))
3728 return;
3729
3730 napi_gro_flush(n);
3731 local_irq_save(flags);
3732 __napi_complete(n);
3733 local_irq_restore(flags);
3734}
3735EXPORT_SYMBOL(napi_complete);
3736
3737void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
3738 int (*poll)(struct napi_struct *, int), int weight)
3739{
3740 INIT_LIST_HEAD(&napi->poll_list);
Herbert Xu4ae55442009-02-08 18:00:36 +00003741 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003742 napi->gro_list = NULL;
Herbert Xu5d38a072009-01-04 16:13:40 -08003743 napi->skb = NULL;
Herbert Xud565b0a2008-12-15 23:38:52 -08003744 napi->poll = poll;
3745 napi->weight = weight;
3746 list_add(&napi->dev_list, &dev->napi_list);
Herbert Xud565b0a2008-12-15 23:38:52 -08003747 napi->dev = dev;
Herbert Xu5d38a072009-01-04 16:13:40 -08003748#ifdef CONFIG_NETPOLL
Herbert Xud565b0a2008-12-15 23:38:52 -08003749 spin_lock_init(&napi->poll_lock);
3750 napi->poll_owner = -1;
3751#endif
3752 set_bit(NAPI_STATE_SCHED, &napi->state);
3753}
3754EXPORT_SYMBOL(netif_napi_add);
3755
3756void netif_napi_del(struct napi_struct *napi)
3757{
3758 struct sk_buff *skb, *next;
3759
Peter P Waskiewicz Jrd7b06632008-12-26 01:35:35 -08003760 list_del_init(&napi->dev_list);
Herbert Xu76620aa2009-04-16 02:02:07 -07003761 napi_free_frags(napi);
Herbert Xud565b0a2008-12-15 23:38:52 -08003762
3763 for (skb = napi->gro_list; skb; skb = next) {
3764 next = skb->next;
3765 skb->next = NULL;
3766 kfree_skb(skb);
3767 }
3768
3769 napi->gro_list = NULL;
Herbert Xu4ae55442009-02-08 18:00:36 +00003770 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003771}
3772EXPORT_SYMBOL(netif_napi_del);
3773
Linus Torvalds1da177e2005-04-16 15:20:36 -07003774static void net_rx_action(struct softirq_action *h)
3775{
Eric Dumazete326bed2010-04-22 00:22:45 -07003776 struct softnet_data *sd = &__get_cpu_var(softnet_data);
Stephen Hemminger24f8b232008-11-03 17:14:38 -08003777 unsigned long time_limit = jiffies + 2;
Stephen Hemminger51b0bde2005-06-23 20:14:40 -07003778 int budget = netdev_budget;
Matt Mackall53fb95d2005-08-11 19:27:43 -07003779 void *have;
3780
Linus Torvalds1da177e2005-04-16 15:20:36 -07003781 local_irq_disable();
3782
Eric Dumazete326bed2010-04-22 00:22:45 -07003783 while (!list_empty(&sd->poll_list)) {
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003784 struct napi_struct *n;
3785 int work, weight;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003786
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003787 /* If softirq window is exhuasted then punt.
Stephen Hemminger24f8b232008-11-03 17:14:38 -08003788 * Allow this to run for 2 jiffies since which will allow
3789 * an average latency of 1.5/HZ.
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003790 */
Stephen Hemminger24f8b232008-11-03 17:14:38 -08003791 if (unlikely(budget <= 0 || time_after(jiffies, time_limit)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003792 goto softnet_break;
3793
3794 local_irq_enable();
3795
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003796 /* Even though interrupts have been re-enabled, this
3797 * access is safe because interrupts can only add new
3798 * entries to the tail of this list, and only ->poll()
3799 * calls can remove this head entry from the list.
3800 */
Eric Dumazete326bed2010-04-22 00:22:45 -07003801 n = list_first_entry(&sd->poll_list, struct napi_struct, poll_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003802
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003803 have = netpoll_poll_lock(n);
3804
3805 weight = n->weight;
3806
David S. Miller0a7606c2007-10-29 21:28:47 -07003807 /* This NAPI_STATE_SCHED test is for avoiding a race
3808 * with netpoll's poll_napi(). Only the entity which
3809 * obtains the lock and sees NAPI_STATE_SCHED set will
3810 * actually make the ->poll() call. Therefore we avoid
Lucas De Marchi25985ed2011-03-30 22:57:33 -03003811 * accidentally calling ->poll() when NAPI is not scheduled.
David S. Miller0a7606c2007-10-29 21:28:47 -07003812 */
3813 work = 0;
Neil Horman4ea7e382009-05-21 07:36:08 +00003814 if (test_bit(NAPI_STATE_SCHED, &n->state)) {
David S. Miller0a7606c2007-10-29 21:28:47 -07003815 work = n->poll(n, weight);
Neil Horman4ea7e382009-05-21 07:36:08 +00003816 trace_napi_poll(n);
3817 }
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003818
3819 WARN_ON_ONCE(work > weight);
3820
3821 budget -= work;
3822
3823 local_irq_disable();
3824
3825 /* Drivers must not modify the NAPI state if they
3826 * consume the entire weight. In such cases this code
3827 * still "owns" the NAPI instance and therefore can
3828 * move the instance around on the list at-will.
3829 */
David S. Millerfed17f32008-01-07 21:00:40 -08003830 if (unlikely(work == weight)) {
Herbert Xuff780cd2009-06-26 19:27:04 -07003831 if (unlikely(napi_disable_pending(n))) {
3832 local_irq_enable();
3833 napi_complete(n);
3834 local_irq_disable();
3835 } else
Eric Dumazete326bed2010-04-22 00:22:45 -07003836 list_move_tail(&n->poll_list, &sd->poll_list);
David S. Millerfed17f32008-01-07 21:00:40 -08003837 }
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003838
3839 netpoll_poll_unlock(have);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003840 }
3841out:
Eric Dumazete326bed2010-04-22 00:22:45 -07003842 net_rps_action_and_irq_enable(sd);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003843
Chris Leechdb217332006-06-17 21:24:58 -07003844#ifdef CONFIG_NET_DMA
3845 /*
3846 * There may not be any more sk_buffs coming right now, so push
3847 * any pending DMA copies to hardware
3848 */
Dan Williams2ba05622009-01-06 11:38:14 -07003849 dma_issue_pending_all();
Chris Leechdb217332006-06-17 21:24:58 -07003850#endif
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003851
Linus Torvalds1da177e2005-04-16 15:20:36 -07003852 return;
3853
3854softnet_break:
Changli Gaodee42872010-05-02 05:42:16 +00003855 sd->time_squeeze++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003856 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3857 goto out;
3858}
3859
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003860static gifconf_func_t *gifconf_list[NPROTO];
Linus Torvalds1da177e2005-04-16 15:20:36 -07003861
3862/**
3863 * register_gifconf - register a SIOCGIF handler
3864 * @family: Address family
3865 * @gifconf: Function handler
3866 *
3867 * Register protocol dependent address dumping routines. The handler
3868 * that is passed must not be freed or reused until it has been replaced
3869 * by another handler.
3870 */
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003871int register_gifconf(unsigned int family, gifconf_func_t *gifconf)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003872{
3873 if (family >= NPROTO)
3874 return -EINVAL;
3875 gifconf_list[family] = gifconf;
3876 return 0;
3877}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003878EXPORT_SYMBOL(register_gifconf);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003879
3880
3881/*
3882 * Map an interface index to its name (SIOCGIFNAME)
3883 */
3884
3885/*
3886 * We need this ioctl for efficient implementation of the
3887 * if_indextoname() function required by the IPv6 API. Without
3888 * it, we would have to search all the interfaces to find a
3889 * match. --pb
3890 */
3891
Eric W. Biederman881d9662007-09-17 11:56:21 -07003892static int dev_ifname(struct net *net, struct ifreq __user *arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003893{
3894 struct net_device *dev;
3895 struct ifreq ifr;
3896
3897 /*
3898 * Fetch the caller's info block.
3899 */
3900
3901 if (copy_from_user(&ifr, arg, sizeof(struct ifreq)))
3902 return -EFAULT;
3903
Eric Dumazetfb699dfd2009-10-19 19:18:49 +00003904 rcu_read_lock();
3905 dev = dev_get_by_index_rcu(net, ifr.ifr_ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003906 if (!dev) {
Eric Dumazetfb699dfd2009-10-19 19:18:49 +00003907 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003908 return -ENODEV;
3909 }
3910
3911 strcpy(ifr.ifr_name, dev->name);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +00003912 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003913
3914 if (copy_to_user(arg, &ifr, sizeof(struct ifreq)))
3915 return -EFAULT;
3916 return 0;
3917}
3918
3919/*
3920 * Perform a SIOCGIFCONF call. This structure will change
3921 * size eventually, and there is nothing I can do about it.
3922 * Thus we will need a 'compatibility mode'.
3923 */
3924
Eric W. Biederman881d9662007-09-17 11:56:21 -07003925static int dev_ifconf(struct net *net, char __user *arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003926{
3927 struct ifconf ifc;
3928 struct net_device *dev;
3929 char __user *pos;
3930 int len;
3931 int total;
3932 int i;
3933
3934 /*
3935 * Fetch the caller's info block.
3936 */
3937
3938 if (copy_from_user(&ifc, arg, sizeof(struct ifconf)))
3939 return -EFAULT;
3940
3941 pos = ifc.ifc_buf;
3942 len = ifc.ifc_len;
3943
3944 /*
3945 * Loop over the interfaces, and write an info block for each.
3946 */
3947
3948 total = 0;
Eric W. Biederman881d9662007-09-17 11:56:21 -07003949 for_each_netdev(net, dev) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003950 for (i = 0; i < NPROTO; i++) {
3951 if (gifconf_list[i]) {
3952 int done;
3953 if (!pos)
3954 done = gifconf_list[i](dev, NULL, 0);
3955 else
3956 done = gifconf_list[i](dev, pos + total,
3957 len - total);
3958 if (done < 0)
3959 return -EFAULT;
3960 total += done;
3961 }
3962 }
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003963 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003964
3965 /*
3966 * All done. Write the updated control block back to the caller.
3967 */
3968 ifc.ifc_len = total;
3969
3970 /*
3971 * Both BSD and Solaris return 0 here, so we do too.
3972 */
3973 return copy_to_user(arg, &ifc, sizeof(struct ifconf)) ? -EFAULT : 0;
3974}
3975
3976#ifdef CONFIG_PROC_FS
3977/*
3978 * This is invoked by the /proc filesystem handler to display a device
3979 * in detail.
3980 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07003981void *dev_seq_start(struct seq_file *seq, loff_t *pos)
Eric Dumazetc6d14c82009-11-04 05:43:23 -08003982 __acquires(RCU)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003983{
Denis V. Luneve372c412007-11-19 22:31:54 -08003984 struct net *net = seq_file_net(seq);
Pavel Emelianov7562f872007-05-03 15:13:45 -07003985 loff_t off;
3986 struct net_device *dev;
3987
Eric Dumazetc6d14c82009-11-04 05:43:23 -08003988 rcu_read_lock();
Pavel Emelianov7562f872007-05-03 15:13:45 -07003989 if (!*pos)
3990 return SEQ_START_TOKEN;
3991
3992 off = 1;
Eric Dumazetc6d14c82009-11-04 05:43:23 -08003993 for_each_netdev_rcu(net, dev)
Pavel Emelianov7562f872007-05-03 15:13:45 -07003994 if (off++ == *pos)
3995 return dev;
3996
3997 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003998}
3999
4000void *dev_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4001{
Eric Dumazetccf43432011-01-26 18:08:02 +00004002 struct net_device *dev = v;
4003
4004 if (v == SEQ_START_TOKEN)
4005 dev = first_net_device_rcu(seq_file_net(seq));
4006 else
4007 dev = next_net_device_rcu(dev);
Eric Dumazetc6d14c82009-11-04 05:43:23 -08004008
Linus Torvalds1da177e2005-04-16 15:20:36 -07004009 ++*pos;
Eric Dumazetccf43432011-01-26 18:08:02 +00004010 return dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004011}
4012
4013void dev_seq_stop(struct seq_file *seq, void *v)
Eric Dumazetc6d14c82009-11-04 05:43:23 -08004014 __releases(RCU)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004015{
Eric Dumazetc6d14c82009-11-04 05:43:23 -08004016 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004017}
4018
4019static void dev_seq_printf_stats(struct seq_file *seq, struct net_device *dev)
4020{
Eric Dumazet28172732010-07-07 14:58:56 -07004021 struct rtnl_link_stats64 temp;
4022 const struct rtnl_link_stats64 *stats = dev_get_stats(dev, &temp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004023
Ben Hutchingsbe1f3c22010-06-08 07:19:54 +00004024 seq_printf(seq, "%6s: %7llu %7llu %4llu %4llu %4llu %5llu %10llu %9llu "
4025 "%8llu %7llu %4llu %4llu %4llu %5llu %7llu %10llu\n",
Rusty Russell5a1b5892007-04-28 21:04:03 -07004026 dev->name, stats->rx_bytes, stats->rx_packets,
4027 stats->rx_errors,
4028 stats->rx_dropped + stats->rx_missed_errors,
4029 stats->rx_fifo_errors,
4030 stats->rx_length_errors + stats->rx_over_errors +
4031 stats->rx_crc_errors + stats->rx_frame_errors,
4032 stats->rx_compressed, stats->multicast,
4033 stats->tx_bytes, stats->tx_packets,
4034 stats->tx_errors, stats->tx_dropped,
4035 stats->tx_fifo_errors, stats->collisions,
4036 stats->tx_carrier_errors +
4037 stats->tx_aborted_errors +
4038 stats->tx_window_errors +
4039 stats->tx_heartbeat_errors,
4040 stats->tx_compressed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004041}
4042
4043/*
4044 * Called from the PROCfs module. This now uses the new arbitrary sized
4045 * /proc/net interface to create /proc/net/dev
4046 */
4047static int dev_seq_show(struct seq_file *seq, void *v)
4048{
4049 if (v == SEQ_START_TOKEN)
4050 seq_puts(seq, "Inter-| Receive "
4051 " | Transmit\n"
4052 " face |bytes packets errs drop fifo frame "
4053 "compressed multicast|bytes packets errs "
4054 "drop fifo colls carrier compressed\n");
4055 else
4056 dev_seq_printf_stats(seq, v);
4057 return 0;
4058}
4059
Changli Gaodee42872010-05-02 05:42:16 +00004060static struct softnet_data *softnet_get_online(loff_t *pos)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004061{
Changli Gaodee42872010-05-02 05:42:16 +00004062 struct softnet_data *sd = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004063
Mike Travis0c0b0ac2008-05-02 16:43:08 -07004064 while (*pos < nr_cpu_ids)
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09004065 if (cpu_online(*pos)) {
Changli Gaodee42872010-05-02 05:42:16 +00004066 sd = &per_cpu(softnet_data, *pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004067 break;
4068 } else
4069 ++*pos;
Changli Gaodee42872010-05-02 05:42:16 +00004070 return sd;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004071}
4072
4073static void *softnet_seq_start(struct seq_file *seq, loff_t *pos)
4074{
4075 return softnet_get_online(pos);
4076}
4077
4078static void *softnet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4079{
4080 ++*pos;
4081 return softnet_get_online(pos);
4082}
4083
4084static void softnet_seq_stop(struct seq_file *seq, void *v)
4085{
4086}
4087
4088static int softnet_seq_show(struct seq_file *seq, void *v)
4089{
Changli Gaodee42872010-05-02 05:42:16 +00004090 struct softnet_data *sd = v;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004091
Tom Herbert0a9627f2010-03-16 08:03:29 +00004092 seq_printf(seq, "%08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n",
Changli Gaodee42872010-05-02 05:42:16 +00004093 sd->processed, sd->dropped, sd->time_squeeze, 0,
Stephen Hemmingerc1ebcdb2005-06-23 20:08:59 -07004094 0, 0, 0, 0, /* was fastroute */
Changli Gaodee42872010-05-02 05:42:16 +00004095 sd->cpu_collision, sd->received_rps);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004096 return 0;
4097}
4098
Stephen Hemmingerf6908082007-03-12 14:34:29 -07004099static const struct seq_operations dev_seq_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004100 .start = dev_seq_start,
4101 .next = dev_seq_next,
4102 .stop = dev_seq_stop,
4103 .show = dev_seq_show,
4104};
4105
4106static int dev_seq_open(struct inode *inode, struct file *file)
4107{
Denis V. Luneve372c412007-11-19 22:31:54 -08004108 return seq_open_net(inode, file, &dev_seq_ops,
4109 sizeof(struct seq_net_private));
Linus Torvalds1da177e2005-04-16 15:20:36 -07004110}
4111
Arjan van de Ven9a321442007-02-12 00:55:35 -08004112static const struct file_operations dev_seq_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004113 .owner = THIS_MODULE,
4114 .open = dev_seq_open,
4115 .read = seq_read,
4116 .llseek = seq_lseek,
Denis V. Luneve372c412007-11-19 22:31:54 -08004117 .release = seq_release_net,
Linus Torvalds1da177e2005-04-16 15:20:36 -07004118};
4119
Stephen Hemmingerf6908082007-03-12 14:34:29 -07004120static const struct seq_operations softnet_seq_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004121 .start = softnet_seq_start,
4122 .next = softnet_seq_next,
4123 .stop = softnet_seq_stop,
4124 .show = softnet_seq_show,
4125};
4126
4127static int softnet_seq_open(struct inode *inode, struct file *file)
4128{
4129 return seq_open(file, &softnet_seq_ops);
4130}
4131
Arjan van de Ven9a321442007-02-12 00:55:35 -08004132static const struct file_operations softnet_seq_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004133 .owner = THIS_MODULE,
4134 .open = softnet_seq_open,
4135 .read = seq_read,
4136 .llseek = seq_lseek,
4137 .release = seq_release,
4138};
4139
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004140static void *ptype_get_idx(loff_t pos)
4141{
4142 struct packet_type *pt = NULL;
4143 loff_t i = 0;
4144 int t;
4145
4146 list_for_each_entry_rcu(pt, &ptype_all, list) {
4147 if (i == pos)
4148 return pt;
4149 ++i;
4150 }
4151
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08004152 for (t = 0; t < PTYPE_HASH_SIZE; t++) {
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004153 list_for_each_entry_rcu(pt, &ptype_base[t], list) {
4154 if (i == pos)
4155 return pt;
4156 ++i;
4157 }
4158 }
4159 return NULL;
4160}
4161
4162static void *ptype_seq_start(struct seq_file *seq, loff_t *pos)
Stephen Hemminger72348a42008-01-21 02:27:29 -08004163 __acquires(RCU)
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004164{
4165 rcu_read_lock();
4166 return *pos ? ptype_get_idx(*pos - 1) : SEQ_START_TOKEN;
4167}
4168
4169static void *ptype_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4170{
4171 struct packet_type *pt;
4172 struct list_head *nxt;
4173 int hash;
4174
4175 ++*pos;
4176 if (v == SEQ_START_TOKEN)
4177 return ptype_get_idx(0);
4178
4179 pt = v;
4180 nxt = pt->list.next;
4181 if (pt->type == htons(ETH_P_ALL)) {
4182 if (nxt != &ptype_all)
4183 goto found;
4184 hash = 0;
4185 nxt = ptype_base[0].next;
4186 } else
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08004187 hash = ntohs(pt->type) & PTYPE_HASH_MASK;
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004188
4189 while (nxt == &ptype_base[hash]) {
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08004190 if (++hash >= PTYPE_HASH_SIZE)
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004191 return NULL;
4192 nxt = ptype_base[hash].next;
4193 }
4194found:
4195 return list_entry(nxt, struct packet_type, list);
4196}
4197
4198static void ptype_seq_stop(struct seq_file *seq, void *v)
Stephen Hemminger72348a42008-01-21 02:27:29 -08004199 __releases(RCU)
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004200{
4201 rcu_read_unlock();
4202}
4203
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004204static int ptype_seq_show(struct seq_file *seq, void *v)
4205{
4206 struct packet_type *pt = v;
4207
4208 if (v == SEQ_START_TOKEN)
4209 seq_puts(seq, "Type Device Function\n");
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09004210 else if (pt->dev == NULL || dev_net(pt->dev) == seq_file_net(seq)) {
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004211 if (pt->type == htons(ETH_P_ALL))
4212 seq_puts(seq, "ALL ");
4213 else
4214 seq_printf(seq, "%04x", ntohs(pt->type));
4215
Alexey Dobriyan908cd2d2008-11-16 19:50:35 -08004216 seq_printf(seq, " %-8s %pF\n",
4217 pt->dev ? pt->dev->name : "", pt->func);
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004218 }
4219
4220 return 0;
4221}
4222
4223static const struct seq_operations ptype_seq_ops = {
4224 .start = ptype_seq_start,
4225 .next = ptype_seq_next,
4226 .stop = ptype_seq_stop,
4227 .show = ptype_seq_show,
4228};
4229
4230static int ptype_seq_open(struct inode *inode, struct file *file)
4231{
Pavel Emelyanov2feb27d2008-03-24 14:57:45 -07004232 return seq_open_net(inode, file, &ptype_seq_ops,
4233 sizeof(struct seq_net_private));
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004234}
4235
4236static const struct file_operations ptype_seq_fops = {
4237 .owner = THIS_MODULE,
4238 .open = ptype_seq_open,
4239 .read = seq_read,
4240 .llseek = seq_lseek,
Pavel Emelyanov2feb27d2008-03-24 14:57:45 -07004241 .release = seq_release_net,
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004242};
4243
4244
Pavel Emelyanov46650792007-10-08 20:38:39 -07004245static int __net_init dev_proc_net_init(struct net *net)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004246{
4247 int rc = -ENOMEM;
4248
Eric W. Biederman881d9662007-09-17 11:56:21 -07004249 if (!proc_net_fops_create(net, "dev", S_IRUGO, &dev_seq_fops))
Linus Torvalds1da177e2005-04-16 15:20:36 -07004250 goto out;
Eric W. Biederman881d9662007-09-17 11:56:21 -07004251 if (!proc_net_fops_create(net, "softnet_stat", S_IRUGO, &softnet_seq_fops))
Linus Torvalds1da177e2005-04-16 15:20:36 -07004252 goto out_dev;
Eric W. Biederman881d9662007-09-17 11:56:21 -07004253 if (!proc_net_fops_create(net, "ptype", S_IRUGO, &ptype_seq_fops))
Eric W. Biederman457c4cb2007-09-12 12:01:34 +02004254 goto out_softnet;
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004255
Eric W. Biederman881d9662007-09-17 11:56:21 -07004256 if (wext_proc_init(net))
Eric W. Biederman457c4cb2007-09-12 12:01:34 +02004257 goto out_ptype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004258 rc = 0;
4259out:
4260 return rc;
Eric W. Biederman457c4cb2007-09-12 12:01:34 +02004261out_ptype:
Eric W. Biederman881d9662007-09-17 11:56:21 -07004262 proc_net_remove(net, "ptype");
Linus Torvalds1da177e2005-04-16 15:20:36 -07004263out_softnet:
Eric W. Biederman881d9662007-09-17 11:56:21 -07004264 proc_net_remove(net, "softnet_stat");
Linus Torvalds1da177e2005-04-16 15:20:36 -07004265out_dev:
Eric W. Biederman881d9662007-09-17 11:56:21 -07004266 proc_net_remove(net, "dev");
Linus Torvalds1da177e2005-04-16 15:20:36 -07004267 goto out;
4268}
Eric W. Biederman881d9662007-09-17 11:56:21 -07004269
Pavel Emelyanov46650792007-10-08 20:38:39 -07004270static void __net_exit dev_proc_net_exit(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07004271{
4272 wext_proc_exit(net);
4273
4274 proc_net_remove(net, "ptype");
4275 proc_net_remove(net, "softnet_stat");
4276 proc_net_remove(net, "dev");
4277}
4278
Denis V. Lunev022cbae2007-11-13 03:23:50 -08004279static struct pernet_operations __net_initdata dev_proc_ops = {
Eric W. Biederman881d9662007-09-17 11:56:21 -07004280 .init = dev_proc_net_init,
4281 .exit = dev_proc_net_exit,
4282};
4283
4284static int __init dev_proc_init(void)
4285{
4286 return register_pernet_subsys(&dev_proc_ops);
4287}
Linus Torvalds1da177e2005-04-16 15:20:36 -07004288#else
4289#define dev_proc_init() 0
4290#endif /* CONFIG_PROC_FS */
4291
4292
4293/**
Jiri Pirko1765a572011-02-12 06:48:36 +00004294 * netdev_set_master - set up master pointer
Linus Torvalds1da177e2005-04-16 15:20:36 -07004295 * @slave: slave device
4296 * @master: new master device
4297 *
4298 * Changes the master device of the slave. Pass %NULL to break the
4299 * bonding. The caller must hold the RTNL semaphore. On a failure
4300 * a negative errno code is returned. On success the reference counts
Jiri Pirko1765a572011-02-12 06:48:36 +00004301 * are adjusted and the function returns zero.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004302 */
4303int netdev_set_master(struct net_device *slave, struct net_device *master)
4304{
4305 struct net_device *old = slave->master;
4306
4307 ASSERT_RTNL();
4308
4309 if (master) {
4310 if (old)
4311 return -EBUSY;
4312 dev_hold(master);
4313 }
4314
4315 slave->master = master;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09004316
Eric Dumazet6df427f2011-05-19 19:37:40 +00004317 if (old)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004318 dev_put(old);
Jiri Pirko1765a572011-02-12 06:48:36 +00004319 return 0;
4320}
4321EXPORT_SYMBOL(netdev_set_master);
4322
4323/**
4324 * netdev_set_bond_master - set up bonding master/slave pair
4325 * @slave: slave device
4326 * @master: new master device
4327 *
4328 * Changes the master device of the slave. Pass %NULL to break the
4329 * bonding. The caller must hold the RTNL semaphore. On a failure
4330 * a negative errno code is returned. On success %RTM_NEWLINK is sent
4331 * to the routing socket and the function returns zero.
4332 */
4333int netdev_set_bond_master(struct net_device *slave, struct net_device *master)
4334{
4335 int err;
4336
4337 ASSERT_RTNL();
4338
4339 err = netdev_set_master(slave, master);
4340 if (err)
4341 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004342 if (master)
4343 slave->flags |= IFF_SLAVE;
4344 else
4345 slave->flags &= ~IFF_SLAVE;
4346
4347 rtmsg_ifinfo(RTM_NEWLINK, slave, IFF_SLAVE);
4348 return 0;
4349}
Jiri Pirko1765a572011-02-12 06:48:36 +00004350EXPORT_SYMBOL(netdev_set_bond_master);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004351
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004352static void dev_change_rx_flags(struct net_device *dev, int flags)
4353{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004354 const struct net_device_ops *ops = dev->netdev_ops;
4355
4356 if ((dev->flags & IFF_UP) && ops->ndo_change_rx_flags)
4357 ops->ndo_change_rx_flags(dev, flags);
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004358}
4359
Wang Chendad9b332008-06-18 01:48:28 -07004360static int __dev_set_promiscuity(struct net_device *dev, int inc)
Patrick McHardy4417da62007-06-27 01:28:10 -07004361{
4362 unsigned short old_flags = dev->flags;
David Howells8192b0c2008-11-14 10:39:10 +11004363 uid_t uid;
4364 gid_t gid;
Patrick McHardy4417da62007-06-27 01:28:10 -07004365
Patrick McHardy24023452007-07-14 18:51:31 -07004366 ASSERT_RTNL();
4367
Wang Chendad9b332008-06-18 01:48:28 -07004368 dev->flags |= IFF_PROMISC;
4369 dev->promiscuity += inc;
4370 if (dev->promiscuity == 0) {
4371 /*
4372 * Avoid overflow.
4373 * If inc causes overflow, untouch promisc and return error.
4374 */
4375 if (inc < 0)
4376 dev->flags &= ~IFF_PROMISC;
4377 else {
4378 dev->promiscuity -= inc;
4379 printk(KERN_WARNING "%s: promiscuity touches roof, "
4380 "set promiscuity failed, promiscuity feature "
4381 "of device might be broken.\n", dev->name);
4382 return -EOVERFLOW;
4383 }
4384 }
Patrick McHardy4417da62007-06-27 01:28:10 -07004385 if (dev->flags != old_flags) {
4386 printk(KERN_INFO "device %s %s promiscuous mode\n",
4387 dev->name, (dev->flags & IFF_PROMISC) ? "entered" :
4388 "left");
David Howells8192b0c2008-11-14 10:39:10 +11004389 if (audit_enabled) {
4390 current_uid_gid(&uid, &gid);
Klaus Heinrich Kiwi7759db82008-01-23 22:57:45 -05004391 audit_log(current->audit_context, GFP_ATOMIC,
4392 AUDIT_ANOM_PROMISCUOUS,
4393 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
4394 dev->name, (dev->flags & IFF_PROMISC),
4395 (old_flags & IFF_PROMISC),
4396 audit_get_loginuid(current),
David Howells8192b0c2008-11-14 10:39:10 +11004397 uid, gid,
Klaus Heinrich Kiwi7759db82008-01-23 22:57:45 -05004398 audit_get_sessionid(current));
David Howells8192b0c2008-11-14 10:39:10 +11004399 }
Patrick McHardy24023452007-07-14 18:51:31 -07004400
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004401 dev_change_rx_flags(dev, IFF_PROMISC);
Patrick McHardy4417da62007-06-27 01:28:10 -07004402 }
Wang Chendad9b332008-06-18 01:48:28 -07004403 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07004404}
4405
Linus Torvalds1da177e2005-04-16 15:20:36 -07004406/**
4407 * dev_set_promiscuity - update promiscuity count on a device
4408 * @dev: device
4409 * @inc: modifier
4410 *
Stephen Hemminger3041a062006-05-26 13:25:24 -07004411 * Add or remove promiscuity from a device. While the count in the device
Linus Torvalds1da177e2005-04-16 15:20:36 -07004412 * remains above zero the interface remains promiscuous. Once it hits zero
4413 * the device reverts back to normal filtering operation. A negative inc
4414 * value is used to drop promiscuity on the device.
Wang Chendad9b332008-06-18 01:48:28 -07004415 * Return 0 if successful or a negative errno code on error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004416 */
Wang Chendad9b332008-06-18 01:48:28 -07004417int dev_set_promiscuity(struct net_device *dev, int inc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004418{
4419 unsigned short old_flags = dev->flags;
Wang Chendad9b332008-06-18 01:48:28 -07004420 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004421
Wang Chendad9b332008-06-18 01:48:28 -07004422 err = __dev_set_promiscuity(dev, inc);
Patrick McHardy4b5a6982008-07-06 15:49:08 -07004423 if (err < 0)
Wang Chendad9b332008-06-18 01:48:28 -07004424 return err;
Patrick McHardy4417da62007-06-27 01:28:10 -07004425 if (dev->flags != old_flags)
4426 dev_set_rx_mode(dev);
Wang Chendad9b332008-06-18 01:48:28 -07004427 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004428}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004429EXPORT_SYMBOL(dev_set_promiscuity);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004430
4431/**
4432 * dev_set_allmulti - update allmulti count on a device
4433 * @dev: device
4434 * @inc: modifier
4435 *
4436 * Add or remove reception of all multicast frames to a device. While the
4437 * count in the device remains above zero the interface remains listening
4438 * to all interfaces. Once it hits zero the device reverts back to normal
4439 * filtering operation. A negative @inc value is used to drop the counter
4440 * when releasing a resource needing all multicasts.
Wang Chendad9b332008-06-18 01:48:28 -07004441 * Return 0 if successful or a negative errno code on error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004442 */
4443
Wang Chendad9b332008-06-18 01:48:28 -07004444int dev_set_allmulti(struct net_device *dev, int inc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004445{
4446 unsigned short old_flags = dev->flags;
4447
Patrick McHardy24023452007-07-14 18:51:31 -07004448 ASSERT_RTNL();
4449
Linus Torvalds1da177e2005-04-16 15:20:36 -07004450 dev->flags |= IFF_ALLMULTI;
Wang Chendad9b332008-06-18 01:48:28 -07004451 dev->allmulti += inc;
4452 if (dev->allmulti == 0) {
4453 /*
4454 * Avoid overflow.
4455 * If inc causes overflow, untouch allmulti and return error.
4456 */
4457 if (inc < 0)
4458 dev->flags &= ~IFF_ALLMULTI;
4459 else {
4460 dev->allmulti -= inc;
4461 printk(KERN_WARNING "%s: allmulti touches roof, "
4462 "set allmulti failed, allmulti feature of "
4463 "device might be broken.\n", dev->name);
4464 return -EOVERFLOW;
4465 }
4466 }
Patrick McHardy24023452007-07-14 18:51:31 -07004467 if (dev->flags ^ old_flags) {
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004468 dev_change_rx_flags(dev, IFF_ALLMULTI);
Patrick McHardy4417da62007-06-27 01:28:10 -07004469 dev_set_rx_mode(dev);
Patrick McHardy24023452007-07-14 18:51:31 -07004470 }
Wang Chendad9b332008-06-18 01:48:28 -07004471 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07004472}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004473EXPORT_SYMBOL(dev_set_allmulti);
Patrick McHardy4417da62007-06-27 01:28:10 -07004474
4475/*
4476 * Upload unicast and multicast address lists to device and
4477 * configure RX filtering. When the device doesn't support unicast
Joe Perches53ccaae2007-12-20 14:02:06 -08004478 * filtering it is put in promiscuous mode while unicast addresses
Patrick McHardy4417da62007-06-27 01:28:10 -07004479 * are present.
4480 */
4481void __dev_set_rx_mode(struct net_device *dev)
4482{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004483 const struct net_device_ops *ops = dev->netdev_ops;
4484
Patrick McHardy4417da62007-06-27 01:28:10 -07004485 /* dev_open will call this function so the list will stay sane. */
4486 if (!(dev->flags&IFF_UP))
4487 return;
4488
4489 if (!netif_device_present(dev))
YOSHIFUJI Hideaki40b77c92007-07-19 10:43:23 +09004490 return;
Patrick McHardy4417da62007-06-27 01:28:10 -07004491
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004492 if (ops->ndo_set_rx_mode)
4493 ops->ndo_set_rx_mode(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07004494 else {
4495 /* Unicast addresses changes may only happen under the rtnl,
4496 * therefore calling __dev_set_promiscuity here is safe.
4497 */
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08004498 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
Patrick McHardy4417da62007-06-27 01:28:10 -07004499 __dev_set_promiscuity(dev, 1);
Joe Perches2d348d12011-07-25 16:17:35 -07004500 dev->uc_promisc = true;
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08004501 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
Patrick McHardy4417da62007-06-27 01:28:10 -07004502 __dev_set_promiscuity(dev, -1);
Joe Perches2d348d12011-07-25 16:17:35 -07004503 dev->uc_promisc = false;
Patrick McHardy4417da62007-06-27 01:28:10 -07004504 }
4505
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004506 if (ops->ndo_set_multicast_list)
4507 ops->ndo_set_multicast_list(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07004508 }
4509}
4510
4511void dev_set_rx_mode(struct net_device *dev)
4512{
David S. Millerb9e40852008-07-15 00:15:08 -07004513 netif_addr_lock_bh(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07004514 __dev_set_rx_mode(dev);
David S. Millerb9e40852008-07-15 00:15:08 -07004515 netif_addr_unlock_bh(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004516}
4517
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004518/**
David Decotigny8ae6dac2011-04-27 18:32:38 +00004519 * dev_ethtool_get_settings - call device's ethtool_ops::get_settings()
4520 * @dev: device
4521 * @cmd: memory area for ethtool_ops::get_settings() result
4522 *
4523 * The cmd arg is initialized properly (cleared and
4524 * ethtool_cmd::cmd field set to ETHTOOL_GSET).
4525 *
4526 * Return device's ethtool_ops::get_settings() result value or
4527 * -EOPNOTSUPP when device doesn't expose
4528 * ethtool_ops::get_settings() operation.
4529 */
4530int dev_ethtool_get_settings(struct net_device *dev,
4531 struct ethtool_cmd *cmd)
4532{
4533 if (!dev->ethtool_ops || !dev->ethtool_ops->get_settings)
4534 return -EOPNOTSUPP;
4535
4536 memset(cmd, 0, sizeof(struct ethtool_cmd));
4537 cmd->cmd = ETHTOOL_GSET;
4538 return dev->ethtool_ops->get_settings(dev, cmd);
4539}
4540EXPORT_SYMBOL(dev_ethtool_get_settings);
4541
4542/**
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004543 * dev_get_flags - get flags reported to userspace
4544 * @dev: device
4545 *
4546 * Get the combination of flag bits exported through APIs to userspace.
4547 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07004548unsigned dev_get_flags(const struct net_device *dev)
4549{
4550 unsigned flags;
4551
4552 flags = (dev->flags & ~(IFF_PROMISC |
4553 IFF_ALLMULTI |
Stefan Rompfb00055a2006-03-20 17:09:11 -08004554 IFF_RUNNING |
4555 IFF_LOWER_UP |
4556 IFF_DORMANT)) |
Linus Torvalds1da177e2005-04-16 15:20:36 -07004557 (dev->gflags & (IFF_PROMISC |
4558 IFF_ALLMULTI));
4559
Stefan Rompfb00055a2006-03-20 17:09:11 -08004560 if (netif_running(dev)) {
4561 if (netif_oper_up(dev))
4562 flags |= IFF_RUNNING;
4563 if (netif_carrier_ok(dev))
4564 flags |= IFF_LOWER_UP;
4565 if (netif_dormant(dev))
4566 flags |= IFF_DORMANT;
4567 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004568
4569 return flags;
4570}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004571EXPORT_SYMBOL(dev_get_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004572
Patrick McHardybd380812010-02-26 06:34:53 +00004573int __dev_change_flags(struct net_device *dev, unsigned int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004574{
Linus Torvalds1da177e2005-04-16 15:20:36 -07004575 int old_flags = dev->flags;
Patrick McHardybd380812010-02-26 06:34:53 +00004576 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004577
Patrick McHardy24023452007-07-14 18:51:31 -07004578 ASSERT_RTNL();
4579
Linus Torvalds1da177e2005-04-16 15:20:36 -07004580 /*
4581 * Set the flags on our device.
4582 */
4583
4584 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
4585 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
4586 IFF_AUTOMEDIA)) |
4587 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
4588 IFF_ALLMULTI));
4589
4590 /*
4591 * Load in the correct multicast list now the flags have changed.
4592 */
4593
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004594 if ((old_flags ^ flags) & IFF_MULTICAST)
4595 dev_change_rx_flags(dev, IFF_MULTICAST);
Patrick McHardy24023452007-07-14 18:51:31 -07004596
Patrick McHardy4417da62007-06-27 01:28:10 -07004597 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004598
4599 /*
4600 * Have we downed the interface. We handle IFF_UP ourselves
4601 * according to user attempts to set it, rather than blindly
4602 * setting it.
4603 */
4604
4605 ret = 0;
4606 if ((old_flags ^ flags) & IFF_UP) { /* Bit is different ? */
Patrick McHardybd380812010-02-26 06:34:53 +00004607 ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004608
4609 if (!ret)
Patrick McHardy4417da62007-06-27 01:28:10 -07004610 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004611 }
4612
Linus Torvalds1da177e2005-04-16 15:20:36 -07004613 if ((flags ^ dev->gflags) & IFF_PROMISC) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004614 int inc = (flags & IFF_PROMISC) ? 1 : -1;
4615
Linus Torvalds1da177e2005-04-16 15:20:36 -07004616 dev->gflags ^= IFF_PROMISC;
4617 dev_set_promiscuity(dev, inc);
4618 }
4619
4620 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
4621 is important. Some (broken) drivers set IFF_PROMISC, when
4622 IFF_ALLMULTI is requested not asking us and not reporting.
4623 */
4624 if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004625 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
4626
Linus Torvalds1da177e2005-04-16 15:20:36 -07004627 dev->gflags ^= IFF_ALLMULTI;
4628 dev_set_allmulti(dev, inc);
4629 }
4630
Patrick McHardybd380812010-02-26 06:34:53 +00004631 return ret;
4632}
4633
4634void __dev_notify_flags(struct net_device *dev, unsigned int old_flags)
4635{
4636 unsigned int changes = dev->flags ^ old_flags;
4637
4638 if (changes & IFF_UP) {
4639 if (dev->flags & IFF_UP)
4640 call_netdevice_notifiers(NETDEV_UP, dev);
4641 else
4642 call_netdevice_notifiers(NETDEV_DOWN, dev);
4643 }
4644
4645 if (dev->flags & IFF_UP &&
4646 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE)))
4647 call_netdevice_notifiers(NETDEV_CHANGE, dev);
4648}
4649
4650/**
4651 * dev_change_flags - change device settings
4652 * @dev: device
4653 * @flags: device state flags
4654 *
4655 * Change settings on device based state flags. The flags are
4656 * in the userspace exported format.
4657 */
4658int dev_change_flags(struct net_device *dev, unsigned flags)
4659{
4660 int ret, changes;
4661 int old_flags = dev->flags;
4662
4663 ret = __dev_change_flags(dev, flags);
4664 if (ret < 0)
4665 return ret;
4666
4667 changes = old_flags ^ dev->flags;
Thomas Graf7c355f52007-06-05 16:03:03 -07004668 if (changes)
4669 rtmsg_ifinfo(RTM_NEWLINK, dev, changes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004670
Patrick McHardybd380812010-02-26 06:34:53 +00004671 __dev_notify_flags(dev, old_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004672 return ret;
4673}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004674EXPORT_SYMBOL(dev_change_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004675
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004676/**
4677 * dev_set_mtu - Change maximum transfer unit
4678 * @dev: device
4679 * @new_mtu: new transfer unit
4680 *
4681 * Change the maximum transfer size of the network device.
4682 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07004683int dev_set_mtu(struct net_device *dev, int new_mtu)
4684{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004685 const struct net_device_ops *ops = dev->netdev_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004686 int err;
4687
4688 if (new_mtu == dev->mtu)
4689 return 0;
4690
4691 /* MTU must be positive. */
4692 if (new_mtu < 0)
4693 return -EINVAL;
4694
4695 if (!netif_device_present(dev))
4696 return -ENODEV;
4697
4698 err = 0;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004699 if (ops->ndo_change_mtu)
4700 err = ops->ndo_change_mtu(dev, new_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004701 else
4702 dev->mtu = new_mtu;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004703
Linus Torvalds1da177e2005-04-16 15:20:36 -07004704 if (!err && dev->flags & IFF_UP)
Pavel Emelyanov056925a2007-09-16 15:42:43 -07004705 call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004706 return err;
4707}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004708EXPORT_SYMBOL(dev_set_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004709
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004710/**
Vlad Dogarucbda10f2011-01-13 23:38:30 +00004711 * dev_set_group - Change group this device belongs to
4712 * @dev: device
4713 * @new_group: group this device should belong to
4714 */
4715void dev_set_group(struct net_device *dev, int new_group)
4716{
4717 dev->group = new_group;
4718}
4719EXPORT_SYMBOL(dev_set_group);
4720
4721/**
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004722 * dev_set_mac_address - Change Media Access Control Address
4723 * @dev: device
4724 * @sa: new address
4725 *
4726 * Change the hardware (MAC) address of the device
4727 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07004728int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
4729{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004730 const struct net_device_ops *ops = dev->netdev_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004731 int err;
4732
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004733 if (!ops->ndo_set_mac_address)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004734 return -EOPNOTSUPP;
4735 if (sa->sa_family != dev->type)
4736 return -EINVAL;
4737 if (!netif_device_present(dev))
4738 return -ENODEV;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004739 err = ops->ndo_set_mac_address(dev, sa);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004740 if (!err)
Pavel Emelyanov056925a2007-09-16 15:42:43 -07004741 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004742 return err;
4743}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004744EXPORT_SYMBOL(dev_set_mac_address);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004745
4746/*
Eric Dumazet3710bec2009-11-01 19:42:09 +00004747 * Perform the SIOCxIFxxx calls, inside rcu_read_lock()
Linus Torvalds1da177e2005-04-16 15:20:36 -07004748 */
Jeff Garzik14e3e072007-10-08 00:06:32 -07004749static int dev_ifsioc_locked(struct net *net, struct ifreq *ifr, unsigned int cmd)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004750{
4751 int err;
Eric Dumazet3710bec2009-11-01 19:42:09 +00004752 struct net_device *dev = dev_get_by_name_rcu(net, ifr->ifr_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004753
4754 if (!dev)
4755 return -ENODEV;
4756
4757 switch (cmd) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004758 case SIOCGIFFLAGS: /* Get interface flags */
4759 ifr->ifr_flags = (short) dev_get_flags(dev);
4760 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004761
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004762 case SIOCGIFMETRIC: /* Get the metric on the interface
4763 (currently unused) */
4764 ifr->ifr_metric = 0;
4765 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004766
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004767 case SIOCGIFMTU: /* Get the MTU of a device */
4768 ifr->ifr_mtu = dev->mtu;
4769 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004770
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004771 case SIOCGIFHWADDR:
4772 if (!dev->addr_len)
4773 memset(ifr->ifr_hwaddr.sa_data, 0, sizeof ifr->ifr_hwaddr.sa_data);
4774 else
4775 memcpy(ifr->ifr_hwaddr.sa_data, dev->dev_addr,
4776 min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len));
4777 ifr->ifr_hwaddr.sa_family = dev->type;
4778 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004779
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004780 case SIOCGIFSLAVE:
4781 err = -EINVAL;
4782 break;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004783
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004784 case SIOCGIFMAP:
4785 ifr->ifr_map.mem_start = dev->mem_start;
4786 ifr->ifr_map.mem_end = dev->mem_end;
4787 ifr->ifr_map.base_addr = dev->base_addr;
4788 ifr->ifr_map.irq = dev->irq;
4789 ifr->ifr_map.dma = dev->dma;
4790 ifr->ifr_map.port = dev->if_port;
4791 return 0;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004792
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004793 case SIOCGIFINDEX:
4794 ifr->ifr_ifindex = dev->ifindex;
4795 return 0;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004796
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004797 case SIOCGIFTXQLEN:
4798 ifr->ifr_qlen = dev->tx_queue_len;
4799 return 0;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004800
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004801 default:
4802 /* dev_ioctl() should ensure this case
4803 * is never reached
4804 */
4805 WARN_ON(1);
Lifeng Sun41c31f32011-04-27 22:04:51 +00004806 err = -ENOTTY;
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004807 break;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004808
4809 }
4810 return err;
4811}
4812
4813/*
4814 * Perform the SIOCxIFxxx calls, inside rtnl_lock()
4815 */
4816static int dev_ifsioc(struct net *net, struct ifreq *ifr, unsigned int cmd)
4817{
4818 int err;
4819 struct net_device *dev = __dev_get_by_name(net, ifr->ifr_name);
Jarek Poplawski5f2f6da2008-12-22 19:35:28 -08004820 const struct net_device_ops *ops;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004821
4822 if (!dev)
4823 return -ENODEV;
4824
Jarek Poplawski5f2f6da2008-12-22 19:35:28 -08004825 ops = dev->netdev_ops;
4826
Jeff Garzik14e3e072007-10-08 00:06:32 -07004827 switch (cmd) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004828 case SIOCSIFFLAGS: /* Set interface flags */
4829 return dev_change_flags(dev, ifr->ifr_flags);
Jeff Garzik14e3e072007-10-08 00:06:32 -07004830
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004831 case SIOCSIFMETRIC: /* Set the metric on the interface
4832 (currently unused) */
4833 return -EOPNOTSUPP;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004834
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004835 case SIOCSIFMTU: /* Set the MTU of a device */
4836 return dev_set_mtu(dev, ifr->ifr_mtu);
Jeff Garzik14e3e072007-10-08 00:06:32 -07004837
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004838 case SIOCSIFHWADDR:
4839 return dev_set_mac_address(dev, &ifr->ifr_hwaddr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004840
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004841 case SIOCSIFHWBROADCAST:
4842 if (ifr->ifr_hwaddr.sa_family != dev->type)
4843 return -EINVAL;
4844 memcpy(dev->broadcast, ifr->ifr_hwaddr.sa_data,
4845 min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len));
4846 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
4847 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004848
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004849 case SIOCSIFMAP:
4850 if (ops->ndo_set_config) {
4851 if (!netif_device_present(dev))
4852 return -ENODEV;
4853 return ops->ndo_set_config(dev, &ifr->ifr_map);
4854 }
4855 return -EOPNOTSUPP;
4856
4857 case SIOCADDMULTI:
4858 if ((!ops->ndo_set_multicast_list && !ops->ndo_set_rx_mode) ||
4859 ifr->ifr_hwaddr.sa_family != AF_UNSPEC)
4860 return -EINVAL;
4861 if (!netif_device_present(dev))
4862 return -ENODEV;
Jiri Pirko22bedad32010-04-01 21:22:57 +00004863 return dev_mc_add_global(dev, ifr->ifr_hwaddr.sa_data);
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004864
4865 case SIOCDELMULTI:
4866 if ((!ops->ndo_set_multicast_list && !ops->ndo_set_rx_mode) ||
4867 ifr->ifr_hwaddr.sa_family != AF_UNSPEC)
4868 return -EINVAL;
4869 if (!netif_device_present(dev))
4870 return -ENODEV;
Jiri Pirko22bedad32010-04-01 21:22:57 +00004871 return dev_mc_del_global(dev, ifr->ifr_hwaddr.sa_data);
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004872
4873 case SIOCSIFTXQLEN:
4874 if (ifr->ifr_qlen < 0)
4875 return -EINVAL;
4876 dev->tx_queue_len = ifr->ifr_qlen;
4877 return 0;
4878
4879 case SIOCSIFNAME:
4880 ifr->ifr_newname[IFNAMSIZ-1] = '\0';
4881 return dev_change_name(dev, ifr->ifr_newname);
4882
4883 /*
4884 * Unknown or private ioctl
4885 */
4886 default:
4887 if ((cmd >= SIOCDEVPRIVATE &&
4888 cmd <= SIOCDEVPRIVATE + 15) ||
4889 cmd == SIOCBONDENSLAVE ||
4890 cmd == SIOCBONDRELEASE ||
4891 cmd == SIOCBONDSETHWADDR ||
4892 cmd == SIOCBONDSLAVEINFOQUERY ||
4893 cmd == SIOCBONDINFOQUERY ||
4894 cmd == SIOCBONDCHANGEACTIVE ||
4895 cmd == SIOCGMIIPHY ||
4896 cmd == SIOCGMIIREG ||
4897 cmd == SIOCSMIIREG ||
4898 cmd == SIOCBRADDIF ||
4899 cmd == SIOCBRDELIF ||
4900 cmd == SIOCSHWTSTAMP ||
4901 cmd == SIOCWANDEV) {
4902 err = -EOPNOTSUPP;
4903 if (ops->ndo_do_ioctl) {
4904 if (netif_device_present(dev))
4905 err = ops->ndo_do_ioctl(dev, ifr, cmd);
4906 else
4907 err = -ENODEV;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004908 }
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004909 } else
4910 err = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004911
4912 }
4913 return err;
4914}
4915
4916/*
4917 * This function handles all "interface"-type I/O control requests. The actual
4918 * 'doing' part of this is dev_ifsioc above.
4919 */
4920
4921/**
4922 * dev_ioctl - network device ioctl
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07004923 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07004924 * @cmd: command to issue
4925 * @arg: pointer to a struct ifreq in user space
4926 *
4927 * Issue ioctl functions to devices. This is normally called by the
4928 * user space syscall interfaces but can sometimes be useful for
4929 * other purposes. The return value is the return from the syscall if
4930 * positive or a negative errno code on error.
4931 */
4932
Eric W. Biederman881d9662007-09-17 11:56:21 -07004933int dev_ioctl(struct net *net, unsigned int cmd, void __user *arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004934{
4935 struct ifreq ifr;
4936 int ret;
4937 char *colon;
4938
4939 /* One special case: SIOCGIFCONF takes ifconf argument
4940 and requires shared lock, because it sleeps writing
4941 to user space.
4942 */
4943
4944 if (cmd == SIOCGIFCONF) {
Stephen Hemminger6756ae42006-03-20 22:23:58 -08004945 rtnl_lock();
Eric W. Biederman881d9662007-09-17 11:56:21 -07004946 ret = dev_ifconf(net, (char __user *) arg);
Stephen Hemminger6756ae42006-03-20 22:23:58 -08004947 rtnl_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004948 return ret;
4949 }
4950 if (cmd == SIOCGIFNAME)
Eric W. Biederman881d9662007-09-17 11:56:21 -07004951 return dev_ifname(net, (struct ifreq __user *)arg);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004952
4953 if (copy_from_user(&ifr, arg, sizeof(struct ifreq)))
4954 return -EFAULT;
4955
4956 ifr.ifr_name[IFNAMSIZ-1] = 0;
4957
4958 colon = strchr(ifr.ifr_name, ':');
4959 if (colon)
4960 *colon = 0;
4961
4962 /*
4963 * See which interface the caller is talking about.
4964 */
4965
4966 switch (cmd) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004967 /*
4968 * These ioctl calls:
4969 * - can be done by all.
4970 * - atomic and do not require locking.
4971 * - return a value
4972 */
4973 case SIOCGIFFLAGS:
4974 case SIOCGIFMETRIC:
4975 case SIOCGIFMTU:
4976 case SIOCGIFHWADDR:
4977 case SIOCGIFSLAVE:
4978 case SIOCGIFMAP:
4979 case SIOCGIFINDEX:
4980 case SIOCGIFTXQLEN:
4981 dev_load(net, ifr.ifr_name);
Eric Dumazet3710bec2009-11-01 19:42:09 +00004982 rcu_read_lock();
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004983 ret = dev_ifsioc_locked(net, &ifr, cmd);
Eric Dumazet3710bec2009-11-01 19:42:09 +00004984 rcu_read_unlock();
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004985 if (!ret) {
4986 if (colon)
4987 *colon = ':';
4988 if (copy_to_user(arg, &ifr,
4989 sizeof(struct ifreq)))
4990 ret = -EFAULT;
4991 }
4992 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004993
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004994 case SIOCETHTOOL:
4995 dev_load(net, ifr.ifr_name);
4996 rtnl_lock();
4997 ret = dev_ethtool(net, &ifr);
4998 rtnl_unlock();
4999 if (!ret) {
5000 if (colon)
5001 *colon = ':';
5002 if (copy_to_user(arg, &ifr,
5003 sizeof(struct ifreq)))
5004 ret = -EFAULT;
5005 }
5006 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005007
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005008 /*
5009 * These ioctl calls:
5010 * - require superuser power.
5011 * - require strict serialization.
5012 * - return a value
5013 */
5014 case SIOCGMIIPHY:
5015 case SIOCGMIIREG:
5016 case SIOCSIFNAME:
5017 if (!capable(CAP_NET_ADMIN))
5018 return -EPERM;
5019 dev_load(net, ifr.ifr_name);
5020 rtnl_lock();
5021 ret = dev_ifsioc(net, &ifr, cmd);
5022 rtnl_unlock();
5023 if (!ret) {
5024 if (colon)
5025 *colon = ':';
5026 if (copy_to_user(arg, &ifr,
5027 sizeof(struct ifreq)))
5028 ret = -EFAULT;
5029 }
5030 return ret;
5031
5032 /*
5033 * These ioctl calls:
5034 * - require superuser power.
5035 * - require strict serialization.
5036 * - do not return a value
5037 */
5038 case SIOCSIFFLAGS:
5039 case SIOCSIFMETRIC:
5040 case SIOCSIFMTU:
5041 case SIOCSIFMAP:
5042 case SIOCSIFHWADDR:
5043 case SIOCSIFSLAVE:
5044 case SIOCADDMULTI:
5045 case SIOCDELMULTI:
5046 case SIOCSIFHWBROADCAST:
5047 case SIOCSIFTXQLEN:
5048 case SIOCSMIIREG:
5049 case SIOCBONDENSLAVE:
5050 case SIOCBONDRELEASE:
5051 case SIOCBONDSETHWADDR:
5052 case SIOCBONDCHANGEACTIVE:
5053 case SIOCBRADDIF:
5054 case SIOCBRDELIF:
5055 case SIOCSHWTSTAMP:
5056 if (!capable(CAP_NET_ADMIN))
5057 return -EPERM;
5058 /* fall through */
5059 case SIOCBONDSLAVEINFOQUERY:
5060 case SIOCBONDINFOQUERY:
5061 dev_load(net, ifr.ifr_name);
5062 rtnl_lock();
5063 ret = dev_ifsioc(net, &ifr, cmd);
5064 rtnl_unlock();
5065 return ret;
5066
5067 case SIOCGIFMEM:
5068 /* Get the per device memory space. We can add this but
5069 * currently do not support it */
5070 case SIOCSIFMEM:
5071 /* Set the per device memory buffer space.
5072 * Not applicable in our case */
5073 case SIOCSIFLINK:
Lifeng Sun41c31f32011-04-27 22:04:51 +00005074 return -ENOTTY;
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005075
5076 /*
5077 * Unknown or private ioctl.
5078 */
5079 default:
5080 if (cmd == SIOCWANDEV ||
5081 (cmd >= SIOCDEVPRIVATE &&
5082 cmd <= SIOCDEVPRIVATE + 15)) {
Eric W. Biederman881d9662007-09-17 11:56:21 -07005083 dev_load(net, ifr.ifr_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005084 rtnl_lock();
Eric W. Biederman881d9662007-09-17 11:56:21 -07005085 ret = dev_ifsioc(net, &ifr, cmd);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005086 rtnl_unlock();
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005087 if (!ret && copy_to_user(arg, &ifr,
Linus Torvalds1da177e2005-04-16 15:20:36 -07005088 sizeof(struct ifreq)))
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005089 ret = -EFAULT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005090 return ret;
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005091 }
5092 /* Take care of Wireless Extensions */
5093 if (cmd >= SIOCIWFIRST && cmd <= SIOCIWLAST)
5094 return wext_handle_ioctl(net, &ifr, cmd, arg);
Lifeng Sun41c31f32011-04-27 22:04:51 +00005095 return -ENOTTY;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005096 }
5097}
5098
5099
5100/**
5101 * dev_new_index - allocate an ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07005102 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07005103 *
5104 * Returns a suitable unique value for a new device interface
5105 * number. The caller must hold the rtnl semaphore or the
5106 * dev_base_lock to be sure it remains unique.
5107 */
Eric W. Biederman881d9662007-09-17 11:56:21 -07005108static int dev_new_index(struct net *net)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005109{
5110 static int ifindex;
5111 for (;;) {
5112 if (++ifindex <= 0)
5113 ifindex = 1;
Eric W. Biederman881d9662007-09-17 11:56:21 -07005114 if (!__dev_get_by_index(net, ifindex))
Linus Torvalds1da177e2005-04-16 15:20:36 -07005115 return ifindex;
5116 }
5117}
5118
Linus Torvalds1da177e2005-04-16 15:20:36 -07005119/* Delayed registration/unregisteration */
Denis Cheng3b5b34f2007-12-07 00:49:17 -08005120static LIST_HEAD(net_todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005121
Stephen Hemminger6f05f622007-03-08 20:46:03 -08005122static void net_set_todo(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005123{
Linus Torvalds1da177e2005-04-16 15:20:36 -07005124 list_add_tail(&dev->todo_list, &net_todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005125}
5126
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005127static void rollback_registered_many(struct list_head *head)
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005128{
Krishna Kumare93737b2009-12-08 22:26:02 +00005129 struct net_device *dev, *tmp;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005130
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005131 BUG_ON(dev_boot_phase);
5132 ASSERT_RTNL();
5133
Krishna Kumare93737b2009-12-08 22:26:02 +00005134 list_for_each_entry_safe(dev, tmp, head, unreg_list) {
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005135 /* Some devices call without registering
Krishna Kumare93737b2009-12-08 22:26:02 +00005136 * for initialization unwind. Remove those
5137 * devices and proceed with the remaining.
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005138 */
5139 if (dev->reg_state == NETREG_UNINITIALIZED) {
5140 pr_debug("unregister_netdevice: device %s/%p never "
5141 "was registered\n", dev->name, dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005142
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005143 WARN_ON(1);
Krishna Kumare93737b2009-12-08 22:26:02 +00005144 list_del(&dev->unreg_list);
5145 continue;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005146 }
Eric Dumazet449f4542011-05-19 12:24:16 +00005147 dev->dismantle = true;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005148 BUG_ON(dev->reg_state != NETREG_REGISTERED);
Octavian Purdila44345722010-12-13 12:44:07 +00005149 }
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005150
Octavian Purdila44345722010-12-13 12:44:07 +00005151 /* If device is running, close it first. */
5152 dev_close_many(head);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005153
Octavian Purdila44345722010-12-13 12:44:07 +00005154 list_for_each_entry(dev, head, unreg_list) {
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005155 /* And unlink it from device chain. */
5156 unlist_netdevice(dev);
5157
5158 dev->reg_state = NETREG_UNREGISTERING;
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005159 }
5160
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005161 synchronize_net();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005162
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005163 list_for_each_entry(dev, head, unreg_list) {
5164 /* Shutdown queueing discipline. */
5165 dev_shutdown(dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005166
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005167
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005168 /* Notify protocols, that we are about to destroy
5169 this device. They should clean all the things.
5170 */
5171 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
5172
Patrick McHardya2835762010-02-26 06:34:51 +00005173 if (!dev->rtnl_link_ops ||
5174 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
5175 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U);
5176
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005177 /*
5178 * Flush the unicast and multicast chains
5179 */
Jiri Pirkoa748ee22010-04-01 21:22:09 +00005180 dev_uc_flush(dev);
Jiri Pirko22bedad32010-04-01 21:22:57 +00005181 dev_mc_flush(dev);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005182
5183 if (dev->netdev_ops->ndo_uninit)
5184 dev->netdev_ops->ndo_uninit(dev);
5185
5186 /* Notifier chain MUST detach us from master device. */
5187 WARN_ON(dev->master);
5188
5189 /* Remove entries from kobject tree */
5190 netdev_unregister_kobject(dev);
5191 }
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005192
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00005193 /* Process any work delayed until the end of the batch */
stephen hemmingere5e26d72010-02-24 14:01:38 +00005194 dev = list_first_entry(head, struct net_device, unreg_list);
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00005195 call_netdevice_notifiers(NETDEV_UNREGISTER_BATCH, dev);
5196
Eric Dumazetef885af2010-09-13 12:24:54 +00005197 rcu_barrier();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005198
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00005199 list_for_each_entry(dev, head, unreg_list)
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005200 dev_put(dev);
5201}
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005202
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005203static void rollback_registered(struct net_device *dev)
5204{
5205 LIST_HEAD(single);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005206
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005207 list_add(&dev->unreg_list, &single);
5208 rollback_registered_many(&single);
Eric Dumazetceaaec92011-02-17 22:59:19 +00005209 list_del(&single);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005210}
5211
Michał Mirosławfec30c32011-07-13 14:10:30 +00005212static u32 netdev_fix_features(struct net_device *dev, u32 features)
Herbert Xub63365a2008-10-23 01:11:29 -07005213{
Michał Mirosław57422dc2011-01-22 12:14:12 +00005214 /* Fix illegal checksum combinations */
5215 if ((features & NETIF_F_HW_CSUM) &&
5216 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005217 netdev_warn(dev, "mixed HW and IP checksum settings.\n");
Michał Mirosław57422dc2011-01-22 12:14:12 +00005218 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
5219 }
5220
5221 if ((features & NETIF_F_NO_CSUM) &&
5222 (features & (NETIF_F_HW_CSUM|NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005223 netdev_warn(dev, "mixed no checksumming and other settings.\n");
Michał Mirosław57422dc2011-01-22 12:14:12 +00005224 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM|NETIF_F_HW_CSUM);
5225 }
5226
Herbert Xub63365a2008-10-23 01:11:29 -07005227 /* Fix illegal SG+CSUM combinations. */
5228 if ((features & NETIF_F_SG) &&
5229 !(features & NETIF_F_ALL_CSUM)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005230 netdev_dbg(dev,
5231 "Dropping NETIF_F_SG since no checksum feature.\n");
Herbert Xub63365a2008-10-23 01:11:29 -07005232 features &= ~NETIF_F_SG;
5233 }
5234
5235 /* TSO requires that SG is present as well. */
Ben Hutchingsea2d3682011-04-12 14:38:37 +00005236 if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005237 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
Ben Hutchingsea2d3682011-04-12 14:38:37 +00005238 features &= ~NETIF_F_ALL_TSO;
Herbert Xub63365a2008-10-23 01:11:29 -07005239 }
5240
Ben Hutchings31d8b9e2011-04-12 14:47:15 +00005241 /* TSO ECN requires that TSO is present as well. */
5242 if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
5243 features &= ~NETIF_F_TSO_ECN;
5244
Michał Mirosław212b5732011-02-15 16:59:16 +00005245 /* Software GSO depends on SG. */
5246 if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005247 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
Michał Mirosław212b5732011-02-15 16:59:16 +00005248 features &= ~NETIF_F_GSO;
5249 }
5250
Michał Mirosławacd11302011-01-24 15:45:15 -08005251 /* UFO needs SG and checksumming */
Herbert Xub63365a2008-10-23 01:11:29 -07005252 if (features & NETIF_F_UFO) {
Michał Mirosław79032642010-11-30 06:38:00 +00005253 /* maybe split UFO into V4 and V6? */
5254 if (!((features & NETIF_F_GEN_CSUM) ||
5255 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))
5256 == (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005257 netdev_dbg(dev,
Michał Mirosławacd11302011-01-24 15:45:15 -08005258 "Dropping NETIF_F_UFO since no checksum offload features.\n");
Herbert Xub63365a2008-10-23 01:11:29 -07005259 features &= ~NETIF_F_UFO;
5260 }
5261
5262 if (!(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005263 netdev_dbg(dev,
Michał Mirosławacd11302011-01-24 15:45:15 -08005264 "Dropping NETIF_F_UFO since no NETIF_F_SG feature.\n");
Herbert Xub63365a2008-10-23 01:11:29 -07005265 features &= ~NETIF_F_UFO;
5266 }
5267 }
5268
5269 return features;
5270}
Herbert Xub63365a2008-10-23 01:11:29 -07005271
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005272int __netdev_update_features(struct net_device *dev)
Michał Mirosław5455c692011-02-15 16:59:17 +00005273{
5274 u32 features;
5275 int err = 0;
5276
Michał Mirosław87267482011-04-12 09:56:38 +00005277 ASSERT_RTNL();
5278
Michał Mirosław5455c692011-02-15 16:59:17 +00005279 features = netdev_get_wanted_features(dev);
5280
5281 if (dev->netdev_ops->ndo_fix_features)
5282 features = dev->netdev_ops->ndo_fix_features(dev, features);
5283
5284 /* driver might be less strict about feature dependencies */
5285 features = netdev_fix_features(dev, features);
5286
5287 if (dev->features == features)
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005288 return 0;
Michał Mirosław5455c692011-02-15 16:59:17 +00005289
Michael S. Tsirkin604ae142011-05-16 10:37:39 +00005290 netdev_dbg(dev, "Features changed: 0x%08x -> 0x%08x\n",
Michał Mirosław5455c692011-02-15 16:59:17 +00005291 dev->features, features);
5292
5293 if (dev->netdev_ops->ndo_set_features)
5294 err = dev->netdev_ops->ndo_set_features(dev, features);
5295
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005296 if (unlikely(err < 0)) {
Michał Mirosław5455c692011-02-15 16:59:17 +00005297 netdev_err(dev,
5298 "set_features() failed (%d); wanted 0x%08x, left 0x%08x\n",
5299 err, features, dev->features);
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005300 return -1;
5301 }
5302
5303 if (!err)
5304 dev->features = features;
5305
5306 return 1;
5307}
5308
Michał Mirosławafe12cc2011-05-07 03:22:17 +00005309/**
5310 * netdev_update_features - recalculate device features
5311 * @dev: the device to check
5312 *
5313 * Recalculate dev->features set and send notifications if it
5314 * has changed. Should be called after driver or hardware dependent
5315 * conditions might have changed that influence the features.
5316 */
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005317void netdev_update_features(struct net_device *dev)
5318{
5319 if (__netdev_update_features(dev))
5320 netdev_features_change(dev);
Michał Mirosław5455c692011-02-15 16:59:17 +00005321}
5322EXPORT_SYMBOL(netdev_update_features);
5323
Linus Torvalds1da177e2005-04-16 15:20:36 -07005324/**
Michał Mirosławafe12cc2011-05-07 03:22:17 +00005325 * netdev_change_features - recalculate device features
5326 * @dev: the device to check
5327 *
5328 * Recalculate dev->features set and send notifications even
5329 * if they have not changed. Should be called instead of
5330 * netdev_update_features() if also dev->vlan_features might
5331 * have changed to allow the changes to be propagated to stacked
5332 * VLAN devices.
5333 */
5334void netdev_change_features(struct net_device *dev)
5335{
5336 __netdev_update_features(dev);
5337 netdev_features_change(dev);
5338}
5339EXPORT_SYMBOL(netdev_change_features);
5340
5341/**
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08005342 * netif_stacked_transfer_operstate - transfer operstate
5343 * @rootdev: the root or lower level device to transfer state from
5344 * @dev: the device to transfer operstate to
5345 *
5346 * Transfer operational state from root to device. This is normally
5347 * called when a stacking relationship exists between the root
5348 * device and the device(a leaf device).
5349 */
5350void netif_stacked_transfer_operstate(const struct net_device *rootdev,
5351 struct net_device *dev)
5352{
5353 if (rootdev->operstate == IF_OPER_DORMANT)
5354 netif_dormant_on(dev);
5355 else
5356 netif_dormant_off(dev);
5357
5358 if (netif_carrier_ok(rootdev)) {
5359 if (!netif_carrier_ok(dev))
5360 netif_carrier_on(dev);
5361 } else {
5362 if (netif_carrier_ok(dev))
5363 netif_carrier_off(dev);
5364 }
5365}
5366EXPORT_SYMBOL(netif_stacked_transfer_operstate);
5367
Tom Herbertbf264142010-11-26 08:36:09 +00005368#ifdef CONFIG_RPS
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005369static int netif_alloc_rx_queues(struct net_device *dev)
5370{
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005371 unsigned int i, count = dev->num_rx_queues;
Tom Herbertbd25fa72010-10-18 18:00:16 +00005372 struct netdev_rx_queue *rx;
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005373
Tom Herbertbd25fa72010-10-18 18:00:16 +00005374 BUG_ON(count < 1);
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005375
Tom Herbertbd25fa72010-10-18 18:00:16 +00005376 rx = kcalloc(count, sizeof(struct netdev_rx_queue), GFP_KERNEL);
5377 if (!rx) {
5378 pr_err("netdev: Unable to allocate %u rx queues.\n", count);
5379 return -ENOMEM;
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005380 }
Tom Herbertbd25fa72010-10-18 18:00:16 +00005381 dev->_rx = rx;
5382
Tom Herbertbd25fa72010-10-18 18:00:16 +00005383 for (i = 0; i < count; i++)
Tom Herbertfe822242010-11-09 10:47:38 +00005384 rx[i].dev = dev;
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005385 return 0;
5386}
Tom Herbertbf264142010-11-26 08:36:09 +00005387#endif
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005388
Changli Gaoaa942102010-12-04 02:31:41 +00005389static void netdev_init_one_queue(struct net_device *dev,
5390 struct netdev_queue *queue, void *_unused)
5391{
5392 /* Initialize queue lock */
5393 spin_lock_init(&queue->_xmit_lock);
5394 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
5395 queue->xmit_lock_owner = -1;
Changli Gaob236da62010-12-14 03:09:15 +00005396 netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
Changli Gaoaa942102010-12-04 02:31:41 +00005397 queue->dev = dev;
5398}
5399
Tom Herberte6484932010-10-18 18:04:39 +00005400static int netif_alloc_netdev_queues(struct net_device *dev)
5401{
5402 unsigned int count = dev->num_tx_queues;
5403 struct netdev_queue *tx;
5404
5405 BUG_ON(count < 1);
5406
5407 tx = kcalloc(count, sizeof(struct netdev_queue), GFP_KERNEL);
5408 if (!tx) {
5409 pr_err("netdev: Unable to allocate %u tx queues.\n",
5410 count);
5411 return -ENOMEM;
5412 }
5413 dev->_tx = tx;
Tom Herbert1d24eb42010-11-21 13:17:27 +00005414
Tom Herberte6484932010-10-18 18:04:39 +00005415 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
5416 spin_lock_init(&dev->tx_global_lock);
Changli Gaoaa942102010-12-04 02:31:41 +00005417
5418 return 0;
Tom Herberte6484932010-10-18 18:04:39 +00005419}
5420
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08005421/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07005422 * register_netdevice - register a network device
5423 * @dev: device to register
5424 *
5425 * Take a completed network device structure and add it to the kernel
5426 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
5427 * chain. 0 is returned on success. A negative errno code is returned
5428 * on a failure to set up the device, or if the name is a duplicate.
5429 *
5430 * Callers must hold the rtnl semaphore. You may want
5431 * register_netdev() instead of this.
5432 *
5433 * BUGS:
5434 * The locking appears insufficient to guarantee two parallel registers
5435 * will not get the same name.
5436 */
5437
5438int register_netdevice(struct net_device *dev)
5439{
Linus Torvalds1da177e2005-04-16 15:20:36 -07005440 int ret;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005441 struct net *net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005442
5443 BUG_ON(dev_boot_phase);
5444 ASSERT_RTNL();
5445
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005446 might_sleep();
5447
Linus Torvalds1da177e2005-04-16 15:20:36 -07005448 /* When net_device's are persistent, this will be fatal. */
5449 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005450 BUG_ON(!net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005451
David S. Millerf1f28aa2008-07-15 00:08:33 -07005452 spin_lock_init(&dev->addr_list_lock);
David S. Millercf508b12008-07-22 14:16:42 -07005453 netdev_set_addr_lockdep_class(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005454
Linus Torvalds1da177e2005-04-16 15:20:36 -07005455 dev->iflink = -1;
5456
Peter Pan(潘卫平)0696c3a2011-05-12 15:46:56 +00005457 ret = dev_get_valid_name(dev, dev->name);
5458 if (ret < 0)
5459 goto out;
5460
Linus Torvalds1da177e2005-04-16 15:20:36 -07005461 /* Init, if this function is available */
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005462 if (dev->netdev_ops->ndo_init) {
5463 ret = dev->netdev_ops->ndo_init(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005464 if (ret) {
5465 if (ret > 0)
5466 ret = -EIO;
Adrian Bunk90833aa2006-11-13 16:02:22 -08005467 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005468 }
5469 }
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005470
Eric W. Biederman881d9662007-09-17 11:56:21 -07005471 dev->ifindex = dev_new_index(net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005472 if (dev->iflink == -1)
5473 dev->iflink = dev->ifindex;
5474
Michał Mirosław5455c692011-02-15 16:59:17 +00005475 /* Transfer changeable features to wanted_features and enable
5476 * software offloads (GSO and GRO).
5477 */
5478 dev->hw_features |= NETIF_F_SOFT_FEATURES;
Michał Mirosław14d12322011-02-22 16:52:28 +00005479 dev->features |= NETIF_F_SOFT_FEATURES;
5480 dev->wanted_features = dev->features & dev->hw_features;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005481
Tom Herbertc6e1a0d2011-04-04 22:30:30 -07005482 /* Turn on no cache copy if HW is doing checksum */
5483 dev->hw_features |= NETIF_F_NOCACHE_COPY;
5484 if ((dev->features & NETIF_F_ALL_CSUM) &&
5485 !(dev->features & NETIF_F_NO_CSUM)) {
5486 dev->wanted_features |= NETIF_F_NOCACHE_COPY;
5487 dev->features |= NETIF_F_NOCACHE_COPY;
5488 }
5489
Michał Mirosław1180e7d2011-07-14 14:41:11 -07005490 /* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
Brandon Philips16c3ea72010-09-15 09:24:24 +00005491 */
Michał Mirosław1180e7d2011-07-14 14:41:11 -07005492 dev->vlan_features |= NETIF_F_HIGHDMA;
Brandon Philips16c3ea72010-09-15 09:24:24 +00005493
Johannes Berg7ffbe3f2009-10-02 05:15:27 +00005494 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
5495 ret = notifier_to_errno(ret);
5496 if (ret)
5497 goto err_uninit;
5498
Eric W. Biederman8b41d182007-09-26 22:02:53 -07005499 ret = netdev_register_kobject(dev);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005500 if (ret)
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07005501 goto err_uninit;
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005502 dev->reg_state = NETREG_REGISTERED;
5503
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005504 __netdev_update_features(dev);
Michał Mirosław8e9b59b2011-02-22 16:52:28 +00005505
Linus Torvalds1da177e2005-04-16 15:20:36 -07005506 /*
5507 * Default initial state at registry is that the
5508 * device is present.
5509 */
5510
5511 set_bit(__LINK_STATE_PRESENT, &dev->state);
5512
Linus Torvalds1da177e2005-04-16 15:20:36 -07005513 dev_init_scheduler(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005514 dev_hold(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02005515 list_netdevice(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005516
5517 /* Notify protocols, that a new device appeared. */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07005518 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07005519 ret = notifier_to_errno(ret);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005520 if (ret) {
5521 rollback_registered(dev);
5522 dev->reg_state = NETREG_UNREGISTERED;
5523 }
Eric W. Biedermand90a9092009-12-12 22:11:15 +00005524 /*
5525 * Prevent userspace races by waiting until the network
5526 * device is fully setup before sending notifications.
5527 */
Patrick McHardya2835762010-02-26 06:34:51 +00005528 if (!dev->rtnl_link_ops ||
5529 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
5530 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005531
5532out:
5533 return ret;
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07005534
5535err_uninit:
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005536 if (dev->netdev_ops->ndo_uninit)
5537 dev->netdev_ops->ndo_uninit(dev);
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07005538 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005539}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005540EXPORT_SYMBOL(register_netdevice);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005541
5542/**
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08005543 * init_dummy_netdev - init a dummy network device for NAPI
5544 * @dev: device to init
5545 *
5546 * This takes a network device structure and initialize the minimum
5547 * amount of fields so it can be used to schedule NAPI polls without
5548 * registering a full blown interface. This is to be used by drivers
5549 * that need to tie several hardware interfaces to a single NAPI
5550 * poll scheduler due to HW limitations.
5551 */
5552int init_dummy_netdev(struct net_device *dev)
5553{
5554 /* Clear everything. Note we don't initialize spinlocks
5555 * are they aren't supposed to be taken by any of the
5556 * NAPI code and this dummy netdev is supposed to be
5557 * only ever used for NAPI polls
5558 */
5559 memset(dev, 0, sizeof(struct net_device));
5560
5561 /* make sure we BUG if trying to hit standard
5562 * register/unregister code path
5563 */
5564 dev->reg_state = NETREG_DUMMY;
5565
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08005566 /* NAPI wants this */
5567 INIT_LIST_HEAD(&dev->napi_list);
5568
5569 /* a dummy interface is started by default */
5570 set_bit(__LINK_STATE_PRESENT, &dev->state);
5571 set_bit(__LINK_STATE_START, &dev->state);
5572
Eric Dumazet29b44332010-10-11 10:22:12 +00005573 /* Note : We dont allocate pcpu_refcnt for dummy devices,
5574 * because users of this 'device' dont need to change
5575 * its refcount.
5576 */
5577
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08005578 return 0;
5579}
5580EXPORT_SYMBOL_GPL(init_dummy_netdev);
5581
5582
5583/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07005584 * register_netdev - register a network device
5585 * @dev: device to register
5586 *
5587 * Take a completed network device structure and add it to the kernel
5588 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
5589 * chain. 0 is returned on success. A negative errno code is returned
5590 * on a failure to set up the device, or if the name is a duplicate.
5591 *
Borislav Petkov38b4da32007-04-20 22:14:10 -07005592 * This is a wrapper around register_netdevice that takes the rtnl semaphore
Linus Torvalds1da177e2005-04-16 15:20:36 -07005593 * and expands the device name if you passed a format string to
5594 * alloc_netdev.
5595 */
5596int register_netdev(struct net_device *dev)
5597{
5598 int err;
5599
5600 rtnl_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005601 err = register_netdevice(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005602 rtnl_unlock();
5603 return err;
5604}
5605EXPORT_SYMBOL(register_netdev);
5606
Eric Dumazet29b44332010-10-11 10:22:12 +00005607int netdev_refcnt_read(const struct net_device *dev)
5608{
5609 int i, refcnt = 0;
5610
5611 for_each_possible_cpu(i)
5612 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
5613 return refcnt;
5614}
5615EXPORT_SYMBOL(netdev_refcnt_read);
5616
Linus Torvalds1da177e2005-04-16 15:20:36 -07005617/*
5618 * netdev_wait_allrefs - wait until all references are gone.
5619 *
5620 * This is called when unregistering network devices.
5621 *
5622 * Any protocol or device that holds a reference should register
5623 * for netdevice notification, and cleanup and put back the
5624 * reference if they receive an UNREGISTER event.
5625 * We can get stuck here if buggy protocols don't correctly
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005626 * call dev_put.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005627 */
5628static void netdev_wait_allrefs(struct net_device *dev)
5629{
5630 unsigned long rebroadcast_time, warning_time;
Eric Dumazet29b44332010-10-11 10:22:12 +00005631 int refcnt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005632
Eric Dumazete014deb2009-11-17 05:59:21 +00005633 linkwatch_forget_dev(dev);
5634
Linus Torvalds1da177e2005-04-16 15:20:36 -07005635 rebroadcast_time = warning_time = jiffies;
Eric Dumazet29b44332010-10-11 10:22:12 +00005636 refcnt = netdev_refcnt_read(dev);
5637
5638 while (refcnt != 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07005639 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
Stephen Hemminger6756ae42006-03-20 22:23:58 -08005640 rtnl_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005641
5642 /* Rebroadcast unregister notification */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07005643 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00005644 /* don't resend NETDEV_UNREGISTER_BATCH, _BATCH users
Octavian Purdila395264d2009-11-16 13:49:35 +00005645 * should have already handle it the first time */
Linus Torvalds1da177e2005-04-16 15:20:36 -07005646
5647 if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
5648 &dev->state)) {
5649 /* We must not have linkwatch events
5650 * pending on unregister. If this
5651 * happens, we simply run the queue
5652 * unscheduled, resulting in a noop
5653 * for this device.
5654 */
5655 linkwatch_run_queue();
5656 }
5657
Stephen Hemminger6756ae42006-03-20 22:23:58 -08005658 __rtnl_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005659
5660 rebroadcast_time = jiffies;
5661 }
5662
5663 msleep(250);
5664
Eric Dumazet29b44332010-10-11 10:22:12 +00005665 refcnt = netdev_refcnt_read(dev);
5666
Linus Torvalds1da177e2005-04-16 15:20:36 -07005667 if (time_after(jiffies, warning_time + 10 * HZ)) {
5668 printk(KERN_EMERG "unregister_netdevice: "
5669 "waiting for %s to become free. Usage "
5670 "count = %d\n",
Eric Dumazet29b44332010-10-11 10:22:12 +00005671 dev->name, refcnt);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005672 warning_time = jiffies;
5673 }
5674 }
5675}
5676
5677/* The sequence is:
5678 *
5679 * rtnl_lock();
5680 * ...
5681 * register_netdevice(x1);
5682 * register_netdevice(x2);
5683 * ...
5684 * unregister_netdevice(y1);
5685 * unregister_netdevice(y2);
5686 * ...
5687 * rtnl_unlock();
5688 * free_netdev(y1);
5689 * free_netdev(y2);
5690 *
Herbert Xu58ec3b42008-10-07 15:50:03 -07005691 * We are invoked by rtnl_unlock().
Linus Torvalds1da177e2005-04-16 15:20:36 -07005692 * This allows us to deal with problems:
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005693 * 1) We can delete sysfs objects which invoke hotplug
Linus Torvalds1da177e2005-04-16 15:20:36 -07005694 * without deadlocking with linkwatch via keventd.
5695 * 2) Since we run with the RTNL semaphore not held, we can sleep
5696 * safely in order to wait for the netdev refcnt to drop to zero.
Herbert Xu58ec3b42008-10-07 15:50:03 -07005697 *
5698 * We must not return until all unregister events added during
5699 * the interval the lock was held have been completed.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005700 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07005701void netdev_run_todo(void)
5702{
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07005703 struct list_head list;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005704
Linus Torvalds1da177e2005-04-16 15:20:36 -07005705 /* Snapshot list, allow later requests */
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07005706 list_replace_init(&net_todo_list, &list);
Herbert Xu58ec3b42008-10-07 15:50:03 -07005707
5708 __rtnl_unlock();
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07005709
Linus Torvalds1da177e2005-04-16 15:20:36 -07005710 while (!list_empty(&list)) {
5711 struct net_device *dev
stephen hemmingere5e26d72010-02-24 14:01:38 +00005712 = list_first_entry(&list, struct net_device, todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005713 list_del(&dev->todo_list);
5714
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005715 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07005716 printk(KERN_ERR "network todo '%s' but state %d\n",
5717 dev->name, dev->reg_state);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005718 dump_stack();
5719 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005720 }
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005721
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005722 dev->reg_state = NETREG_UNREGISTERED;
5723
Changli Gao152102c2010-03-30 20:16:22 +00005724 on_each_cpu(flush_backlog, dev, 1);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07005725
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005726 netdev_wait_allrefs(dev);
5727
5728 /* paranoia */
Eric Dumazet29b44332010-10-11 10:22:12 +00005729 BUG_ON(netdev_refcnt_read(dev));
Eric Dumazet95ae6b22010-09-15 04:04:31 +00005730 WARN_ON(rcu_dereference_raw(dev->ip_ptr));
Eric Dumazet198caec2010-10-24 21:32:05 +00005731 WARN_ON(rcu_dereference_raw(dev->ip6_ptr));
Ilpo Järvinen547b7922008-07-25 21:43:18 -07005732 WARN_ON(dev->dn_ptr);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005733
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005734 if (dev->destructor)
5735 dev->destructor(dev);
Stephen Hemminger9093bbb2007-05-19 15:39:25 -07005736
5737 /* Free network device */
5738 kobject_put(&dev->dev.kobj);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005739 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005740}
5741
Ben Hutchings3cfde792010-07-09 09:11:52 +00005742/* Convert net_device_stats to rtnl_link_stats64. They have the same
5743 * fields in the same order, with only the type differing.
5744 */
5745static void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
5746 const struct net_device_stats *netdev_stats)
5747{
5748#if BITS_PER_LONG == 64
5749 BUILD_BUG_ON(sizeof(*stats64) != sizeof(*netdev_stats));
5750 memcpy(stats64, netdev_stats, sizeof(*stats64));
5751#else
5752 size_t i, n = sizeof(*stats64) / sizeof(u64);
5753 const unsigned long *src = (const unsigned long *)netdev_stats;
5754 u64 *dst = (u64 *)stats64;
5755
5756 BUILD_BUG_ON(sizeof(*netdev_stats) / sizeof(unsigned long) !=
5757 sizeof(*stats64) / sizeof(u64));
5758 for (i = 0; i < n; i++)
5759 dst[i] = src[i];
5760#endif
5761}
5762
Eric Dumazetd83345a2009-11-16 03:36:51 +00005763/**
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005764 * dev_get_stats - get network device statistics
5765 * @dev: device to get statistics from
Eric Dumazet28172732010-07-07 14:58:56 -07005766 * @storage: place to store stats
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005767 *
Ben Hutchingsd7753512010-07-09 09:12:41 +00005768 * Get network statistics from device. Return @storage.
5769 * The device driver may provide its own method by setting
5770 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
5771 * otherwise the internal statistics structure is used.
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005772 */
Ben Hutchingsd7753512010-07-09 09:12:41 +00005773struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
5774 struct rtnl_link_stats64 *storage)
Eric Dumazet7004bf22009-05-18 00:34:33 +00005775{
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005776 const struct net_device_ops *ops = dev->netdev_ops;
5777
Eric Dumazet28172732010-07-07 14:58:56 -07005778 if (ops->ndo_get_stats64) {
5779 memset(storage, 0, sizeof(*storage));
Eric Dumazetcaf586e2010-09-30 21:06:55 +00005780 ops->ndo_get_stats64(dev, storage);
5781 } else if (ops->ndo_get_stats) {
Ben Hutchings3cfde792010-07-09 09:11:52 +00005782 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
Eric Dumazetcaf586e2010-09-30 21:06:55 +00005783 } else {
5784 netdev_stats_to_stats64(storage, &dev->stats);
Eric Dumazet28172732010-07-07 14:58:56 -07005785 }
Eric Dumazetcaf586e2010-09-30 21:06:55 +00005786 storage->rx_dropped += atomic_long_read(&dev->rx_dropped);
Eric Dumazet28172732010-07-07 14:58:56 -07005787 return storage;
Rusty Russellc45d2862007-03-28 14:29:08 -07005788}
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005789EXPORT_SYMBOL(dev_get_stats);
Rusty Russellc45d2862007-03-28 14:29:08 -07005790
Eric Dumazet24824a02010-10-02 06:11:55 +00005791struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
David S. Millerdc2b4842008-07-08 17:18:23 -07005792{
Eric Dumazet24824a02010-10-02 06:11:55 +00005793 struct netdev_queue *queue = dev_ingress_queue(dev);
David S. Millerdc2b4842008-07-08 17:18:23 -07005794
Eric Dumazet24824a02010-10-02 06:11:55 +00005795#ifdef CONFIG_NET_CLS_ACT
5796 if (queue)
5797 return queue;
5798 queue = kzalloc(sizeof(*queue), GFP_KERNEL);
5799 if (!queue)
5800 return NULL;
5801 netdev_init_one_queue(dev, queue, NULL);
Eric Dumazet24824a02010-10-02 06:11:55 +00005802 queue->qdisc = &noop_qdisc;
5803 queue->qdisc_sleeping = &noop_qdisc;
5804 rcu_assign_pointer(dev->ingress_queue, queue);
5805#endif
5806 return queue;
David S. Millerbb949fb2008-07-08 16:55:56 -07005807}
5808
Linus Torvalds1da177e2005-04-16 15:20:36 -07005809/**
Tom Herbert36909ea2011-01-09 19:36:31 +00005810 * alloc_netdev_mqs - allocate network device
Linus Torvalds1da177e2005-04-16 15:20:36 -07005811 * @sizeof_priv: size of private data to allocate space for
5812 * @name: device name format string
5813 * @setup: callback to initialize device
Tom Herbert36909ea2011-01-09 19:36:31 +00005814 * @txqs: the number of TX subqueues to allocate
5815 * @rxqs: the number of RX subqueues to allocate
Linus Torvalds1da177e2005-04-16 15:20:36 -07005816 *
5817 * Allocates a struct net_device with private data area for driver use
Peter P Waskiewicz Jrf25f4e42007-07-06 13:36:20 -07005818 * and performs basic initialization. Also allocates subquue structs
Tom Herbert36909ea2011-01-09 19:36:31 +00005819 * for each queue on the device.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005820 */
Tom Herbert36909ea2011-01-09 19:36:31 +00005821struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
5822 void (*setup)(struct net_device *),
5823 unsigned int txqs, unsigned int rxqs)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005824{
Linus Torvalds1da177e2005-04-16 15:20:36 -07005825 struct net_device *dev;
Stephen Hemminger79439862008-07-21 13:28:44 -07005826 size_t alloc_size;
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00005827 struct net_device *p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005828
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -07005829 BUG_ON(strlen(name) >= sizeof(dev->name));
5830
Tom Herbert36909ea2011-01-09 19:36:31 +00005831 if (txqs < 1) {
Tom Herbert55513fb2010-10-18 17:55:58 +00005832 pr_err("alloc_netdev: Unable to allocate device "
5833 "with zero queues.\n");
5834 return NULL;
5835 }
5836
Tom Herbert36909ea2011-01-09 19:36:31 +00005837#ifdef CONFIG_RPS
5838 if (rxqs < 1) {
5839 pr_err("alloc_netdev: Unable to allocate device "
5840 "with zero RX queues.\n");
5841 return NULL;
5842 }
5843#endif
5844
David S. Millerfd2ea0a2008-07-17 01:56:23 -07005845 alloc_size = sizeof(struct net_device);
Alexey Dobriyand1643d22008-04-18 15:43:32 -07005846 if (sizeof_priv) {
5847 /* ensure 32-byte alignment of private area */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00005848 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
Alexey Dobriyand1643d22008-04-18 15:43:32 -07005849 alloc_size += sizeof_priv;
5850 }
5851 /* ensure 32-byte alignment of whole construct */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00005852 alloc_size += NETDEV_ALIGN - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005853
Paolo 'Blaisorblade' Giarrusso31380de2006-04-06 22:38:28 -07005854 p = kzalloc(alloc_size, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005855 if (!p) {
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -07005856 printk(KERN_ERR "alloc_netdev: Unable to allocate device.\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07005857 return NULL;
5858 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005859
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00005860 dev = PTR_ALIGN(p, NETDEV_ALIGN);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005861 dev->padded = (char *)dev - (char *)p;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00005862
Eric Dumazet29b44332010-10-11 10:22:12 +00005863 dev->pcpu_refcnt = alloc_percpu(int);
5864 if (!dev->pcpu_refcnt)
Tom Herberte6484932010-10-18 18:04:39 +00005865 goto free_p;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00005866
Linus Torvalds1da177e2005-04-16 15:20:36 -07005867 if (dev_addr_init(dev))
Eric Dumazet29b44332010-10-11 10:22:12 +00005868 goto free_pcpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005869
Jiri Pirko22bedad32010-04-01 21:22:57 +00005870 dev_mc_init(dev);
Jiri Pirkoa748ee22010-04-01 21:22:09 +00005871 dev_uc_init(dev);
Jiri Pirkoccffad252009-05-22 23:22:17 +00005872
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09005873 dev_net_set(dev, &init_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005874
Peter P Waskiewicz Jr82cc1a72008-03-21 03:43:19 -07005875 dev->gso_max_size = GSO_MAX_SIZE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005876
Herbert Xud565b0a2008-12-15 23:38:52 -08005877 INIT_LIST_HEAD(&dev->napi_list);
Eric W. Biederman9fdce092009-10-30 14:51:13 +00005878 INIT_LIST_HEAD(&dev->unreg_list);
Eric Dumazete014deb2009-11-17 05:59:21 +00005879 INIT_LIST_HEAD(&dev->link_watch_list);
Eric Dumazet93f154b2009-05-18 22:19:19 -07005880 dev->priv_flags = IFF_XMIT_DST_RELEASE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005881 setup(dev);
David S. Miller8d3bdbd2011-02-08 15:02:50 -08005882
5883 dev->num_tx_queues = txqs;
5884 dev->real_num_tx_queues = txqs;
5885 if (netif_alloc_netdev_queues(dev))
5886 goto free_all;
5887
5888#ifdef CONFIG_RPS
5889 dev->num_rx_queues = rxqs;
5890 dev->real_num_rx_queues = rxqs;
5891 if (netif_alloc_rx_queues(dev))
5892 goto free_all;
5893#endif
5894
Linus Torvalds1da177e2005-04-16 15:20:36 -07005895 strcpy(dev->name, name);
Vlad Dogarucbda10f2011-01-13 23:38:30 +00005896 dev->group = INIT_NETDEV_GROUP;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005897 return dev;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00005898
David S. Miller8d3bdbd2011-02-08 15:02:50 -08005899free_all:
5900 free_netdev(dev);
5901 return NULL;
5902
Eric Dumazet29b44332010-10-11 10:22:12 +00005903free_pcpu:
5904 free_percpu(dev->pcpu_refcnt);
Tom Herberted9af2e2010-11-09 10:47:30 +00005905 kfree(dev->_tx);
Tom Herbertfe822242010-11-09 10:47:38 +00005906#ifdef CONFIG_RPS
5907 kfree(dev->_rx);
5908#endif
5909
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00005910free_p:
5911 kfree(p);
5912 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005913}
Tom Herbert36909ea2011-01-09 19:36:31 +00005914EXPORT_SYMBOL(alloc_netdev_mqs);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005915
5916/**
5917 * free_netdev - free network device
5918 * @dev: device
5919 *
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005920 * This function does the last stage of destroying an allocated device
5921 * interface. The reference to the device object is released.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005922 * If this is the last reference then it will be freed.
5923 */
5924void free_netdev(struct net_device *dev)
5925{
Herbert Xud565b0a2008-12-15 23:38:52 -08005926 struct napi_struct *p, *n;
5927
Denis V. Lunevf3005d72008-04-16 02:02:18 -07005928 release_net(dev_net(dev));
5929
David S. Millere8a04642008-07-17 00:34:19 -07005930 kfree(dev->_tx);
Tom Herbertfe822242010-11-09 10:47:38 +00005931#ifdef CONFIG_RPS
5932 kfree(dev->_rx);
5933#endif
David S. Millere8a04642008-07-17 00:34:19 -07005934
Eric Dumazet24824a02010-10-02 06:11:55 +00005935 kfree(rcu_dereference_raw(dev->ingress_queue));
5936
Jiri Pirkof001fde2009-05-05 02:48:28 +00005937 /* Flush device addresses */
5938 dev_addr_flush(dev);
5939
Herbert Xud565b0a2008-12-15 23:38:52 -08005940 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
5941 netif_napi_del(p);
5942
Eric Dumazet29b44332010-10-11 10:22:12 +00005943 free_percpu(dev->pcpu_refcnt);
5944 dev->pcpu_refcnt = NULL;
5945
Stephen Hemminger3041a062006-05-26 13:25:24 -07005946 /* Compatibility with error handling in drivers */
Linus Torvalds1da177e2005-04-16 15:20:36 -07005947 if (dev->reg_state == NETREG_UNINITIALIZED) {
5948 kfree((char *)dev - dev->padded);
5949 return;
5950 }
5951
5952 BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
5953 dev->reg_state = NETREG_RELEASED;
5954
Greg Kroah-Hartman43cb76d2002-04-09 12:14:34 -07005955 /* will free via device release */
5956 put_device(&dev->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005957}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005958EXPORT_SYMBOL(free_netdev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005959
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07005960/**
5961 * synchronize_net - Synchronize with packet receive processing
5962 *
5963 * Wait for packets currently being received to be done.
5964 * Does not block later packets from starting.
5965 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005966void synchronize_net(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005967{
5968 might_sleep();
Eric Dumazetbe3fc412011-05-23 23:07:32 +00005969 if (rtnl_is_locked())
5970 synchronize_rcu_expedited();
5971 else
5972 synchronize_rcu();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005973}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005974EXPORT_SYMBOL(synchronize_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005975
5976/**
Eric Dumazet44a08732009-10-27 07:03:04 +00005977 * unregister_netdevice_queue - remove device from the kernel
Linus Torvalds1da177e2005-04-16 15:20:36 -07005978 * @dev: device
Eric Dumazet44a08732009-10-27 07:03:04 +00005979 * @head: list
Jaswinder Singh Rajput6ebfbc02009-11-22 20:43:13 -08005980 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07005981 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08005982 * from the kernel tables.
Eric Dumazet44a08732009-10-27 07:03:04 +00005983 * If head not NULL, device is queued to be unregistered later.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005984 *
5985 * Callers must hold the rtnl semaphore. You may want
5986 * unregister_netdev() instead of this.
5987 */
5988
Eric Dumazet44a08732009-10-27 07:03:04 +00005989void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005990{
Herbert Xua6620712007-12-12 19:21:56 -08005991 ASSERT_RTNL();
5992
Eric Dumazet44a08732009-10-27 07:03:04 +00005993 if (head) {
Eric W. Biederman9fdce092009-10-30 14:51:13 +00005994 list_move_tail(&dev->unreg_list, head);
Eric Dumazet44a08732009-10-27 07:03:04 +00005995 } else {
5996 rollback_registered(dev);
5997 /* Finish processing unregister after unlock */
5998 net_set_todo(dev);
5999 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006000}
Eric Dumazet44a08732009-10-27 07:03:04 +00006001EXPORT_SYMBOL(unregister_netdevice_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006002
6003/**
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006004 * unregister_netdevice_many - unregister many devices
6005 * @head: list of devices
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006006 */
6007void unregister_netdevice_many(struct list_head *head)
6008{
6009 struct net_device *dev;
6010
6011 if (!list_empty(head)) {
6012 rollback_registered_many(head);
6013 list_for_each_entry(dev, head, unreg_list)
6014 net_set_todo(dev);
6015 }
6016}
Eric Dumazet63c80992009-10-27 07:06:49 +00006017EXPORT_SYMBOL(unregister_netdevice_many);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006018
6019/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07006020 * unregister_netdev - remove device from the kernel
6021 * @dev: device
6022 *
6023 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08006024 * from the kernel tables.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006025 *
6026 * This is just a wrapper for unregister_netdevice that takes
6027 * the rtnl semaphore. In general you want to use this and not
6028 * unregister_netdevice.
6029 */
6030void unregister_netdev(struct net_device *dev)
6031{
6032 rtnl_lock();
6033 unregister_netdevice(dev);
6034 rtnl_unlock();
6035}
Linus Torvalds1da177e2005-04-16 15:20:36 -07006036EXPORT_SYMBOL(unregister_netdev);
6037
Eric W. Biedermance286d32007-09-12 13:53:49 +02006038/**
6039 * dev_change_net_namespace - move device to different nethost namespace
6040 * @dev: device
6041 * @net: network namespace
6042 * @pat: If not NULL name pattern to try if the current device name
6043 * is already taken in the destination network namespace.
6044 *
6045 * This function shuts down a device interface and moves it
6046 * to a new network namespace. On success 0 is returned, on
6047 * a failure a netagive errno code is returned.
6048 *
6049 * Callers must hold the rtnl semaphore.
6050 */
6051
6052int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
6053{
Eric W. Biedermance286d32007-09-12 13:53:49 +02006054 int err;
6055
6056 ASSERT_RTNL();
6057
6058 /* Don't allow namespace local devices to be moved. */
6059 err = -EINVAL;
6060 if (dev->features & NETIF_F_NETNS_LOCAL)
6061 goto out;
6062
6063 /* Ensure the device has been registrered */
6064 err = -EINVAL;
6065 if (dev->reg_state != NETREG_REGISTERED)
6066 goto out;
6067
6068 /* Get out if there is nothing todo */
6069 err = 0;
YOSHIFUJI Hideaki878628f2008-03-26 03:57:35 +09006070 if (net_eq(dev_net(dev), net))
Eric W. Biedermance286d32007-09-12 13:53:49 +02006071 goto out;
6072
6073 /* Pick the destination device name, and ensure
6074 * we can use it in the destination network namespace.
6075 */
6076 err = -EEXIST;
Octavian Purdilad9031022009-11-18 02:36:59 +00006077 if (__dev_get_by_name(net, dev->name)) {
Eric W. Biedermance286d32007-09-12 13:53:49 +02006078 /* We get here if we can't use the current device name */
6079 if (!pat)
6080 goto out;
Jiri Pirko1c5cae82011-04-30 01:21:32 +00006081 if (dev_get_valid_name(dev, pat) < 0)
Eric W. Biedermance286d32007-09-12 13:53:49 +02006082 goto out;
6083 }
6084
6085 /*
6086 * And now a mini version of register_netdevice unregister_netdevice.
6087 */
6088
6089 /* If device is running close it first. */
Pavel Emelyanov9b772652007-10-10 02:49:09 -07006090 dev_close(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006091
6092 /* And unlink it from device chain */
6093 err = -ENODEV;
6094 unlist_netdevice(dev);
6095
6096 synchronize_net();
6097
6098 /* Shutdown queueing discipline. */
6099 dev_shutdown(dev);
6100
6101 /* Notify protocols, that we are about to destroy
6102 this device. They should clean all the things.
David Lamparter3b27e102010-09-17 03:22:19 +00006103
6104 Note that dev->reg_state stays at NETREG_REGISTERED.
6105 This is wanted because this way 8021q and macvlan know
6106 the device is just moving and can keep their slaves up.
Eric W. Biedermance286d32007-09-12 13:53:49 +02006107 */
6108 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00006109 call_netdevice_notifiers(NETDEV_UNREGISTER_BATCH, dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006110
6111 /*
6112 * Flush the unicast and multicast chains
6113 */
Jiri Pirkoa748ee22010-04-01 21:22:09 +00006114 dev_uc_flush(dev);
Jiri Pirko22bedad32010-04-01 21:22:57 +00006115 dev_mc_flush(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006116
6117 /* Actually switch the network namespace */
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09006118 dev_net_set(dev, net);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006119
Eric W. Biedermance286d32007-09-12 13:53:49 +02006120 /* If there is an ifindex conflict assign a new one */
6121 if (__dev_get_by_index(net, dev->ifindex)) {
6122 int iflink = (dev->iflink == dev->ifindex);
6123 dev->ifindex = dev_new_index(net);
6124 if (iflink)
6125 dev->iflink = dev->ifindex;
6126 }
6127
Eric W. Biederman8b41d182007-09-26 22:02:53 -07006128 /* Fixup kobjects */
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07006129 err = device_rename(&dev->dev, dev->name);
Eric W. Biederman8b41d182007-09-26 22:02:53 -07006130 WARN_ON(err);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006131
6132 /* Add the device back in the hashes */
6133 list_netdevice(dev);
6134
6135 /* Notify protocols, that a new device appeared. */
6136 call_netdevice_notifiers(NETDEV_REGISTER, dev);
6137
Eric W. Biedermand90a9092009-12-12 22:11:15 +00006138 /*
6139 * Prevent userspace races by waiting until the network
6140 * device is fully setup before sending notifications.
6141 */
6142 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);
6143
Eric W. Biedermance286d32007-09-12 13:53:49 +02006144 synchronize_net();
6145 err = 0;
6146out:
6147 return err;
6148}
Johannes Berg463d0182009-07-14 00:33:35 +02006149EXPORT_SYMBOL_GPL(dev_change_net_namespace);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006150
Linus Torvalds1da177e2005-04-16 15:20:36 -07006151static int dev_cpu_callback(struct notifier_block *nfb,
6152 unsigned long action,
6153 void *ocpu)
6154{
6155 struct sk_buff **list_skb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006156 struct sk_buff *skb;
6157 unsigned int cpu, oldcpu = (unsigned long)ocpu;
6158 struct softnet_data *sd, *oldsd;
6159
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07006160 if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006161 return NOTIFY_OK;
6162
6163 local_irq_disable();
6164 cpu = smp_processor_id();
6165 sd = &per_cpu(softnet_data, cpu);
6166 oldsd = &per_cpu(softnet_data, oldcpu);
6167
6168 /* Find end of our completion_queue. */
6169 list_skb = &sd->completion_queue;
6170 while (*list_skb)
6171 list_skb = &(*list_skb)->next;
6172 /* Append completion queue from offline CPU. */
6173 *list_skb = oldsd->completion_queue;
6174 oldsd->completion_queue = NULL;
6175
Linus Torvalds1da177e2005-04-16 15:20:36 -07006176 /* Append output queue from offline CPU. */
Changli Gaoa9cbd582010-04-26 23:06:24 +00006177 if (oldsd->output_queue) {
6178 *sd->output_queue_tailp = oldsd->output_queue;
6179 sd->output_queue_tailp = oldsd->output_queue_tailp;
6180 oldsd->output_queue = NULL;
6181 oldsd->output_queue_tailp = &oldsd->output_queue;
6182 }
Heiko Carstens264524d2011-06-06 20:50:03 +00006183 /* Append NAPI poll list from offline CPU. */
6184 if (!list_empty(&oldsd->poll_list)) {
6185 list_splice_init(&oldsd->poll_list, &sd->poll_list);
6186 raise_softirq_irqoff(NET_RX_SOFTIRQ);
6187 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006188
6189 raise_softirq_irqoff(NET_TX_SOFTIRQ);
6190 local_irq_enable();
6191
6192 /* Process offline CPU's input_pkt_queue */
Tom Herbert76cc8b12010-05-20 18:37:59 +00006193 while ((skb = __skb_dequeue(&oldsd->process_queue))) {
6194 netif_rx(skb);
6195 input_queue_head_incr(oldsd);
6196 }
Tom Herbertfec5e652010-04-16 16:01:27 -07006197 while ((skb = __skb_dequeue(&oldsd->input_pkt_queue))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07006198 netif_rx(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00006199 input_queue_head_incr(oldsd);
Tom Herbertfec5e652010-04-16 16:01:27 -07006200 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006201
6202 return NOTIFY_OK;
6203}
Linus Torvalds1da177e2005-04-16 15:20:36 -07006204
6205
Herbert Xu7f353bf2007-08-10 15:47:58 -07006206/**
Herbert Xub63365a2008-10-23 01:11:29 -07006207 * netdev_increment_features - increment feature set by one
6208 * @all: current feature set
6209 * @one: new feature set
6210 * @mask: mask feature set
Herbert Xu7f353bf2007-08-10 15:47:58 -07006211 *
6212 * Computes a new feature set after adding a device with feature set
Herbert Xub63365a2008-10-23 01:11:29 -07006213 * @one to the master device with current feature set @all. Will not
6214 * enable anything that is off in @mask. Returns the new feature set.
Herbert Xu7f353bf2007-08-10 15:47:58 -07006215 */
Michał Mirosław04ed3e72011-01-24 15:32:47 -08006216u32 netdev_increment_features(u32 all, u32 one, u32 mask)
Herbert Xu7f353bf2007-08-10 15:47:58 -07006217{
Michał Mirosław1742f182011-04-22 06:31:16 +00006218 if (mask & NETIF_F_GEN_CSUM)
6219 mask |= NETIF_F_ALL_CSUM;
6220 mask |= NETIF_F_VLAN_CHALLENGED;
6221
6222 all |= one & (NETIF_F_ONE_FOR_ALL|NETIF_F_ALL_CSUM) & mask;
6223 all &= one | ~NETIF_F_ALL_FOR_ALL;
6224
Herbert Xub63365a2008-10-23 01:11:29 -07006225 /* If device needs checksumming, downgrade to it. */
Michał Mirosław1742f182011-04-22 06:31:16 +00006226 if (all & (NETIF_F_ALL_CSUM & ~NETIF_F_NO_CSUM))
6227 all &= ~NETIF_F_NO_CSUM;
Herbert Xu7f353bf2007-08-10 15:47:58 -07006228
Michał Mirosław1742f182011-04-22 06:31:16 +00006229 /* If one device supports hw checksumming, set for all. */
6230 if (all & NETIF_F_GEN_CSUM)
6231 all &= ~(NETIF_F_ALL_CSUM & ~NETIF_F_GEN_CSUM);
Herbert Xu7f353bf2007-08-10 15:47:58 -07006232
6233 return all;
6234}
Herbert Xub63365a2008-10-23 01:11:29 -07006235EXPORT_SYMBOL(netdev_increment_features);
Herbert Xu7f353bf2007-08-10 15:47:58 -07006236
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07006237static struct hlist_head *netdev_create_hash(void)
6238{
6239 int i;
6240 struct hlist_head *hash;
6241
6242 hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
6243 if (hash != NULL)
6244 for (i = 0; i < NETDEV_HASHENTRIES; i++)
6245 INIT_HLIST_HEAD(&hash[i]);
6246
6247 return hash;
6248}
6249
Eric W. Biederman881d9662007-09-17 11:56:21 -07006250/* Initialize per network namespace state */
Pavel Emelyanov46650792007-10-08 20:38:39 -07006251static int __net_init netdev_init(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07006252{
Eric W. Biederman881d9662007-09-17 11:56:21 -07006253 INIT_LIST_HEAD(&net->dev_base_head);
Eric W. Biederman881d9662007-09-17 11:56:21 -07006254
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07006255 net->dev_name_head = netdev_create_hash();
6256 if (net->dev_name_head == NULL)
6257 goto err_name;
Eric W. Biederman881d9662007-09-17 11:56:21 -07006258
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07006259 net->dev_index_head = netdev_create_hash();
6260 if (net->dev_index_head == NULL)
6261 goto err_idx;
Eric W. Biederman881d9662007-09-17 11:56:21 -07006262
6263 return 0;
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07006264
6265err_idx:
6266 kfree(net->dev_name_head);
6267err_name:
6268 return -ENOMEM;
Eric W. Biederman881d9662007-09-17 11:56:21 -07006269}
6270
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006271/**
6272 * netdev_drivername - network driver for the device
6273 * @dev: network device
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006274 *
6275 * Determine network driver for device.
6276 */
David S. Miller3019de12011-06-06 16:41:33 -07006277const char *netdev_drivername(const struct net_device *dev)
Arjan van de Ven6579e572008-07-21 13:31:48 -07006278{
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -07006279 const struct device_driver *driver;
6280 const struct device *parent;
David S. Miller3019de12011-06-06 16:41:33 -07006281 const char *empty = "";
Arjan van de Ven6579e572008-07-21 13:31:48 -07006282
6283 parent = dev->dev.parent;
Arjan van de Ven6579e572008-07-21 13:31:48 -07006284 if (!parent)
David S. Miller3019de12011-06-06 16:41:33 -07006285 return empty;
Arjan van de Ven6579e572008-07-21 13:31:48 -07006286
6287 driver = parent->driver;
6288 if (driver && driver->name)
David S. Miller3019de12011-06-06 16:41:33 -07006289 return driver->name;
6290 return empty;
Arjan van de Ven6579e572008-07-21 13:31:48 -07006291}
6292
Joe Perches256df2f2010-06-27 01:02:35 +00006293static int __netdev_printk(const char *level, const struct net_device *dev,
6294 struct va_format *vaf)
6295{
6296 int r;
6297
6298 if (dev && dev->dev.parent)
6299 r = dev_printk(level, dev->dev.parent, "%s: %pV",
6300 netdev_name(dev), vaf);
6301 else if (dev)
6302 r = printk("%s%s: %pV", level, netdev_name(dev), vaf);
6303 else
6304 r = printk("%s(NULL net_device): %pV", level, vaf);
6305
6306 return r;
6307}
6308
6309int netdev_printk(const char *level, const struct net_device *dev,
6310 const char *format, ...)
6311{
6312 struct va_format vaf;
6313 va_list args;
6314 int r;
6315
6316 va_start(args, format);
6317
6318 vaf.fmt = format;
6319 vaf.va = &args;
6320
6321 r = __netdev_printk(level, dev, &vaf);
6322 va_end(args);
6323
6324 return r;
6325}
6326EXPORT_SYMBOL(netdev_printk);
6327
6328#define define_netdev_printk_level(func, level) \
6329int func(const struct net_device *dev, const char *fmt, ...) \
6330{ \
6331 int r; \
6332 struct va_format vaf; \
6333 va_list args; \
6334 \
6335 va_start(args, fmt); \
6336 \
6337 vaf.fmt = fmt; \
6338 vaf.va = &args; \
6339 \
6340 r = __netdev_printk(level, dev, &vaf); \
6341 va_end(args); \
6342 \
6343 return r; \
6344} \
6345EXPORT_SYMBOL(func);
6346
6347define_netdev_printk_level(netdev_emerg, KERN_EMERG);
6348define_netdev_printk_level(netdev_alert, KERN_ALERT);
6349define_netdev_printk_level(netdev_crit, KERN_CRIT);
6350define_netdev_printk_level(netdev_err, KERN_ERR);
6351define_netdev_printk_level(netdev_warn, KERN_WARNING);
6352define_netdev_printk_level(netdev_notice, KERN_NOTICE);
6353define_netdev_printk_level(netdev_info, KERN_INFO);
6354
Pavel Emelyanov46650792007-10-08 20:38:39 -07006355static void __net_exit netdev_exit(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07006356{
6357 kfree(net->dev_name_head);
6358 kfree(net->dev_index_head);
6359}
6360
Denis V. Lunev022cbae2007-11-13 03:23:50 -08006361static struct pernet_operations __net_initdata netdev_net_ops = {
Eric W. Biederman881d9662007-09-17 11:56:21 -07006362 .init = netdev_init,
6363 .exit = netdev_exit,
6364};
6365
Pavel Emelyanov46650792007-10-08 20:38:39 -07006366static void __net_exit default_device_exit(struct net *net)
Eric W. Biedermance286d32007-09-12 13:53:49 +02006367{
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006368 struct net_device *dev, *aux;
Eric W. Biedermance286d32007-09-12 13:53:49 +02006369 /*
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006370 * Push all migratable network devices back to the
Eric W. Biedermance286d32007-09-12 13:53:49 +02006371 * initial network namespace
6372 */
6373 rtnl_lock();
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006374 for_each_netdev_safe(net, dev, aux) {
Eric W. Biedermance286d32007-09-12 13:53:49 +02006375 int err;
Pavel Emelyanovaca51392008-05-08 01:24:25 -07006376 char fb_name[IFNAMSIZ];
Eric W. Biedermance286d32007-09-12 13:53:49 +02006377
6378 /* Ignore unmoveable devices (i.e. loopback) */
6379 if (dev->features & NETIF_F_NETNS_LOCAL)
6380 continue;
6381
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006382 /* Leave virtual devices for the generic cleanup */
6383 if (dev->rtnl_link_ops)
6384 continue;
Eric W. Biedermand0c082c2008-11-05 15:59:38 -08006385
Lucas De Marchi25985ed2011-03-30 22:57:33 -03006386 /* Push remaining network devices to init_net */
Pavel Emelyanovaca51392008-05-08 01:24:25 -07006387 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
6388 err = dev_change_net_namespace(dev, &init_net, fb_name);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006389 if (err) {
Pavel Emelyanovaca51392008-05-08 01:24:25 -07006390 printk(KERN_EMERG "%s: failed to move %s to init_net: %d\n",
Eric W. Biedermance286d32007-09-12 13:53:49 +02006391 __func__, dev->name, err);
Pavel Emelyanovaca51392008-05-08 01:24:25 -07006392 BUG();
Eric W. Biedermance286d32007-09-12 13:53:49 +02006393 }
6394 }
6395 rtnl_unlock();
6396}
6397
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00006398static void __net_exit default_device_exit_batch(struct list_head *net_list)
6399{
6400 /* At exit all network devices most be removed from a network
Uwe Kleine-Königb5950762010-11-01 15:38:34 -04006401 * namespace. Do this in the reverse order of registration.
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00006402 * Do this across as many network namespaces as possible to
6403 * improve batching efficiency.
6404 */
6405 struct net_device *dev;
6406 struct net *net;
6407 LIST_HEAD(dev_kill_list);
6408
6409 rtnl_lock();
6410 list_for_each_entry(net, net_list, exit_list) {
6411 for_each_netdev_reverse(net, dev) {
6412 if (dev->rtnl_link_ops)
6413 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
6414 else
6415 unregister_netdevice_queue(dev, &dev_kill_list);
6416 }
6417 }
6418 unregister_netdevice_many(&dev_kill_list);
Eric Dumazetceaaec92011-02-17 22:59:19 +00006419 list_del(&dev_kill_list);
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00006420 rtnl_unlock();
6421}
6422
Denis V. Lunev022cbae2007-11-13 03:23:50 -08006423static struct pernet_operations __net_initdata default_device_ops = {
Eric W. Biedermance286d32007-09-12 13:53:49 +02006424 .exit = default_device_exit,
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00006425 .exit_batch = default_device_exit_batch,
Eric W. Biedermance286d32007-09-12 13:53:49 +02006426};
6427
Linus Torvalds1da177e2005-04-16 15:20:36 -07006428/*
6429 * Initialize the DEV module. At boot time this walks the device list and
6430 * unhooks any devices that fail to initialise (normally hardware not
6431 * present) and leaves us with a valid list of present and active devices.
6432 *
6433 */
6434
6435/*
6436 * This is called single threaded during boot, so no need
6437 * to take the rtnl semaphore.
6438 */
6439static int __init net_dev_init(void)
6440{
6441 int i, rc = -ENOMEM;
6442
6443 BUG_ON(!dev_boot_phase);
6444
Linus Torvalds1da177e2005-04-16 15:20:36 -07006445 if (dev_proc_init())
6446 goto out;
6447
Eric W. Biederman8b41d182007-09-26 22:02:53 -07006448 if (netdev_kobject_init())
Linus Torvalds1da177e2005-04-16 15:20:36 -07006449 goto out;
6450
6451 INIT_LIST_HEAD(&ptype_all);
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08006452 for (i = 0; i < PTYPE_HASH_SIZE; i++)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006453 INIT_LIST_HEAD(&ptype_base[i]);
6454
Eric W. Biederman881d9662007-09-17 11:56:21 -07006455 if (register_pernet_subsys(&netdev_net_ops))
6456 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006457
6458 /*
6459 * Initialise the packet receive queues.
6460 */
6461
KAMEZAWA Hiroyuki6f912042006-04-10 22:52:50 -07006462 for_each_possible_cpu(i) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006463 struct softnet_data *sd = &per_cpu(softnet_data, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006464
Changli Gaodee42872010-05-02 05:42:16 +00006465 memset(sd, 0, sizeof(*sd));
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006466 skb_queue_head_init(&sd->input_pkt_queue);
Changli Gao6e7676c2010-04-27 15:07:33 -07006467 skb_queue_head_init(&sd->process_queue);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006468 sd->completion_queue = NULL;
6469 INIT_LIST_HEAD(&sd->poll_list);
Changli Gaoa9cbd582010-04-26 23:06:24 +00006470 sd->output_queue = NULL;
6471 sd->output_queue_tailp = &sd->output_queue;
Eric Dumazetdf334542010-03-24 19:13:54 +00006472#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006473 sd->csd.func = rps_trigger_softirq;
6474 sd->csd.info = sd;
6475 sd->csd.flags = 0;
6476 sd->cpu = i;
Tom Herbert1e94d722010-03-18 17:45:44 -07006477#endif
Tom Herbert0a9627f2010-03-16 08:03:29 +00006478
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006479 sd->backlog.poll = process_backlog;
6480 sd->backlog.weight = weight_p;
6481 sd->backlog.gro_list = NULL;
6482 sd->backlog.gro_count = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006483 }
6484
Linus Torvalds1da177e2005-04-16 15:20:36 -07006485 dev_boot_phase = 0;
6486
Eric W. Biederman505d4f72008-11-07 22:54:20 -08006487 /* The loopback device is special if any other network devices
6488 * is present in a network namespace the loopback device must
6489 * be present. Since we now dynamically allocate and free the
6490 * loopback device ensure this invariant is maintained by
6491 * keeping the loopback device as the first device on the
6492 * list of network devices. Ensuring the loopback devices
6493 * is the first device that appears and the last network device
6494 * that disappears.
6495 */
6496 if (register_pernet_device(&loopback_net_ops))
6497 goto out;
6498
6499 if (register_pernet_device(&default_device_ops))
6500 goto out;
6501
Carlos R. Mafra962cf362008-05-15 11:15:37 -03006502 open_softirq(NET_TX_SOFTIRQ, net_tx_action);
6503 open_softirq(NET_RX_SOFTIRQ, net_rx_action);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006504
6505 hotcpu_notifier(dev_cpu_callback, 0);
6506 dst_init();
6507 dev_mcast_init();
6508 rc = 0;
6509out:
6510 return rc;
6511}
6512
6513subsys_initcall(net_dev_init);
6514
Krishna Kumare88721f2009-02-18 17:55:02 -08006515static int __init initialize_hashrnd(void)
6516{
Tom Herbert0a9627f2010-03-16 08:03:29 +00006517 get_random_bytes(&hashrnd, sizeof(hashrnd));
Krishna Kumare88721f2009-02-18 17:55:02 -08006518 return 0;
6519}
6520
6521late_initcall_sync(initialize_hashrnd);
6522