blob: b7a7f5a721bddae6fc56231e184558cbb6bc6906 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * PACKET - implements raw packet sockets.
7 *
Jesper Juhl02c30a82005-05-05 16:16:16 -07008 * Authors: Ross Biro
Linus Torvalds1da177e2005-04-16 15:20:36 -07009 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
11 *
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +090012 * Fixes:
Linus Torvalds1da177e2005-04-16 15:20:36 -070013 * Alan Cox : verify_area() now used correctly
14 * Alan Cox : new skbuff lists, look ma no backlogs!
15 * Alan Cox : tidied skbuff lists.
16 * Alan Cox : Now uses generic datagram routines I
17 * added. Also fixed the peek/read crash
18 * from all old Linux datagram code.
19 * Alan Cox : Uses the improved datagram code.
20 * Alan Cox : Added NULL's for socket options.
21 * Alan Cox : Re-commented the code.
22 * Alan Cox : Use new kernel side addressing
23 * Rob Janssen : Correct MTU usage.
24 * Dave Platt : Counter leaks caused by incorrect
25 * interrupt locking and some slightly
26 * dubious gcc output. Can you read
27 * compiler: it said _VOLATILE_
28 * Richard Kooijman : Timestamp fixes.
29 * Alan Cox : New buffers. Use sk->mac.raw.
30 * Alan Cox : sendmsg/recvmsg support.
31 * Alan Cox : Protocol setting support
32 * Alexey Kuznetsov : Untied from IPv4 stack.
33 * Cyrus Durgin : Fixed kerneld for kmod.
34 * Michal Ostrowski : Module initialization cleanup.
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +090035 * Ulises Alonso : Frame number limit removal and
Linus Torvalds1da177e2005-04-16 15:20:36 -070036 * packet_set_ring memory leak.
Eric W. Biederman0fb375f2005-09-21 00:11:37 -070037 * Eric Biederman : Allow for > 8 byte hardware addresses.
38 * The convention is that longer addresses
39 * will simply extend the hardware address
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +090040 * byte arrays at the end of sockaddr_ll
Eric W. Biederman0fb375f2005-09-21 00:11:37 -070041 * and packet_mreq.
Johann Baudy69e3c752009-05-18 22:11:22 -070042 * Johann Baudy : Added TX RING.
chetan lokef6fb8f102011-08-19 10:18:16 +000043 * Chetan Loke : Implemented TPACKET_V3 block abstraction
44 * layer.
45 * Copyright (C) 2011, <lokec@ccs.neu.edu>
46 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070047 *
48 * This program is free software; you can redistribute it and/or
49 * modify it under the terms of the GNU General Public License
50 * as published by the Free Software Foundation; either version
51 * 2 of the License, or (at your option) any later version.
52 *
53 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +090054
Linus Torvalds1da177e2005-04-16 15:20:36 -070055#include <linux/types.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070056#include <linux/mm.h>
Randy Dunlap4fc268d2006-01-11 12:17:47 -080057#include <linux/capability.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070058#include <linux/fcntl.h>
59#include <linux/socket.h>
60#include <linux/in.h>
61#include <linux/inet.h>
62#include <linux/netdevice.h>
63#include <linux/if_packet.h>
64#include <linux/wireless.h>
Herbert Xuffbc6112007-02-04 23:33:10 -080065#include <linux/kernel.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070066#include <linux/kmod.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090067#include <linux/slab.h>
Neil Horman0e3125c2010-11-16 10:26:47 -080068#include <linux/vmalloc.h>
Eric W. Biederman457c4cb2007-09-12 12:01:34 +020069#include <net/net_namespace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070070#include <net/ip.h>
71#include <net/protocol.h>
72#include <linux/skbuff.h>
73#include <net/sock.h>
74#include <linux/errno.h>
75#include <linux/timer.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070076#include <asm/uaccess.h>
77#include <asm/ioctls.h>
78#include <asm/page.h>
Al Viroa1f8e7f72006-10-19 16:08:53 -040079#include <asm/cacheflush.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070080#include <asm/io.h>
81#include <linux/proc_fs.h>
82#include <linux/seq_file.h>
83#include <linux/poll.h>
84#include <linux/module.h>
85#include <linux/init.h>
Herbert Xu905db442009-01-30 14:12:06 -080086#include <linux/mutex.h>
Eric Dumazet05423b22009-10-26 18:40:35 -070087#include <linux/if_vlan.h>
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -080088#include <linux/virtio_net.h>
Richard Cochraned85b562010-04-07 22:41:28 +000089#include <linux/errqueue.h>
Scott McMillan614f60f2010-06-02 05:53:56 -070090#include <linux/net_tstamp.h>
Daniel Borkmannb0138402014-01-15 16:25:36 +010091#include <linux/percpu.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070092#ifdef CONFIG_INET
93#include <net/inet_common.h>
94#endif
95
Pavel Emelyanov2787b042012-08-13 05:49:39 +000096#include "internal.h"
97
Linus Torvalds1da177e2005-04-16 15:20:36 -070098/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070099 Assumptions:
100 - if device has no dev->hard_header routine, it adds and removes ll header
101 inside itself. In this case ll header is invisible outside of device,
102 but higher levels still should reserve dev->hard_header_len.
103 Some devices are enough clever to reallocate skb, when header
104 will not fit to reserved space (tunnel), another ones are silly
105 (PPP).
106 - packet socket receives packets with pulled ll header,
107 so that SOCK_RAW should push it back.
108
109On receive:
110-----------
111
112Incoming, dev->hard_header!=NULL
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -0700113 mac_header -> ll header
114 data -> data
Linus Torvalds1da177e2005-04-16 15:20:36 -0700115
116Outgoing, dev->hard_header!=NULL
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -0700117 mac_header -> ll header
118 data -> ll header
Linus Torvalds1da177e2005-04-16 15:20:36 -0700119
120Incoming, dev->hard_header==NULL
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -0700121 mac_header -> UNKNOWN position. It is very likely, that it points to ll
122 header. PPP makes it, that is wrong, because introduce
YOSHIFUJI Hideakidb0c58f2007-07-19 10:44:35 +0900123 assymetry between rx and tx paths.
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -0700124 data -> data
Linus Torvalds1da177e2005-04-16 15:20:36 -0700125
126Outgoing, dev->hard_header==NULL
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -0700127 mac_header -> data. ll header is still not built!
128 data -> data
Linus Torvalds1da177e2005-04-16 15:20:36 -0700129
130Resume
131 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
132
133
134On transmit:
135------------
136
137dev->hard_header != NULL
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -0700138 mac_header -> ll header
139 data -> ll header
Linus Torvalds1da177e2005-04-16 15:20:36 -0700140
141dev->hard_header == NULL (ll header is added by device, we cannot control it)
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -0700142 mac_header -> data
143 data -> data
Linus Torvalds1da177e2005-04-16 15:20:36 -0700144
145 We should set nh.raw on output to correct posistion,
146 packet classifier depends on it.
147 */
148
Linus Torvalds1da177e2005-04-16 15:20:36 -0700149/* Private packet socket structures. */
150
Eric W. Biederman0fb375f2005-09-21 00:11:37 -0700151/* identical to struct packet_mreq except it has
152 * a longer address field.
153 */
Eric Dumazet40d4e3d2009-07-21 21:57:59 +0000154struct packet_mreq_max {
Eric W. Biederman0fb375f2005-09-21 00:11:37 -0700155 int mr_ifindex;
156 unsigned short mr_type;
157 unsigned short mr_alen;
158 unsigned char mr_address[MAX_ADDR_LEN];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700159};
David S. Millera2efcfa2007-05-29 13:12:50 -0700160
Daniel Borkmann184f4892013-04-16 01:57:46 +0000161union tpacket_uhdr {
162 struct tpacket_hdr *h1;
163 struct tpacket2_hdr *h2;
164 struct tpacket3_hdr *h3;
165 void *raw;
166};
167
chetan lokef6fb8f102011-08-19 10:18:16 +0000168static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
Johann Baudy69e3c752009-05-18 22:11:22 -0700169 int closing, int tx_ring);
170
chetan lokef6fb8f102011-08-19 10:18:16 +0000171#define V3_ALIGNMENT (8)
172
chetan lokebc59ba32011-08-25 10:43:30 +0000173#define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
chetan lokef6fb8f102011-08-19 10:18:16 +0000174
175#define BLK_PLUS_PRIV(sz_of_priv) \
176 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
177
chetan lokef6fb8f102011-08-19 10:18:16 +0000178#define PGV_FROM_VMALLOC 1
Johann Baudy69e3c752009-05-18 22:11:22 -0700179
chetan lokef6fb8f102011-08-19 10:18:16 +0000180#define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
181#define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
182#define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
183#define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
184#define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
185#define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
186#define BLOCK_PRIV(x) ((void *)((char *)(x) + BLOCK_O2PRIV(x)))
187
Johann Baudy69e3c752009-05-18 22:11:22 -0700188struct packet_sock;
189static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
Willem de Bruijn77f65eb2013-03-19 10:18:11 +0000190static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
191 struct packet_type *pt, struct net_device *orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700192
chetan lokef6fb8f102011-08-19 10:18:16 +0000193static void *packet_previous_frame(struct packet_sock *po,
194 struct packet_ring_buffer *rb,
195 int status);
196static void packet_increment_head(struct packet_ring_buffer *buff);
chetan lokebc59ba32011-08-25 10:43:30 +0000197static int prb_curr_blk_in_use(struct tpacket_kbdq_core *,
198 struct tpacket_block_desc *);
199static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
chetan lokef6fb8f102011-08-19 10:18:16 +0000200 struct packet_sock *);
chetan lokebc59ba32011-08-25 10:43:30 +0000201static void prb_retire_current_block(struct tpacket_kbdq_core *,
chetan lokef6fb8f102011-08-19 10:18:16 +0000202 struct packet_sock *, unsigned int status);
chetan lokebc59ba32011-08-25 10:43:30 +0000203static int prb_queue_frozen(struct tpacket_kbdq_core *);
204static void prb_open_block(struct tpacket_kbdq_core *,
205 struct tpacket_block_desc *);
chetan lokef6fb8f102011-08-19 10:18:16 +0000206static void prb_retire_rx_blk_timer_expired(unsigned long);
chetan lokebc59ba32011-08-25 10:43:30 +0000207static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
208static void prb_init_blk_timer(struct packet_sock *,
209 struct tpacket_kbdq_core *,
210 void (*func) (unsigned long));
211static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
212static void prb_clear_rxhash(struct tpacket_kbdq_core *,
213 struct tpacket3_hdr *);
214static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
215 struct tpacket3_hdr *);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700216static void packet_flush_mclist(struct sock *sk);
217
Herbert Xuffbc6112007-02-04 23:33:10 -0800218struct packet_skb_cb {
219 unsigned int origlen;
220 union {
221 struct sockaddr_pkt pkt;
222 struct sockaddr_ll ll;
223 } sa;
224};
225
226#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
Herbert Xu8dc41942007-02-04 23:31:32 -0800227
chetan lokebc59ba32011-08-25 10:43:30 +0000228#define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
chetan lokef6fb8f102011-08-19 10:18:16 +0000229#define GET_PBLOCK_DESC(x, bid) \
chetan lokebc59ba32011-08-25 10:43:30 +0000230 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
chetan lokef6fb8f102011-08-19 10:18:16 +0000231#define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
chetan lokebc59ba32011-08-25 10:43:30 +0000232 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
chetan lokef6fb8f102011-08-19 10:18:16 +0000233#define GET_NEXT_PRB_BLK_NUM(x) \
234 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
235 ((x)->kactive_blk_num+1) : 0)
236
David S. Millerdc99f602011-07-05 01:45:05 -0700237static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
238static void __fanout_link(struct sock *sk, struct packet_sock *po);
239
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100240static int packet_direct_xmit(struct sk_buff *skb)
241{
242 struct net_device *dev = skb->dev;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100243 netdev_features_t features;
244 struct netdev_queue *txq;
Daniel Borkmann43279502014-03-27 16:38:30 +0100245 int ret = NETDEV_TX_BUSY;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100246
247 if (unlikely(!netif_running(dev) ||
Daniel Borkmann43279502014-03-27 16:38:30 +0100248 !netif_carrier_ok(dev)))
249 goto drop;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100250
251 features = netif_skb_features(skb);
252 if (skb_needs_linearize(skb, features) &&
Daniel Borkmann43279502014-03-27 16:38:30 +0100253 __skb_linearize(skb))
254 goto drop;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100255
Daniel Borkmann10c51b56232014-08-27 11:11:27 +0200256 txq = skb_get_tx_queue(dev, skb);
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100257
Daniel Borkmann43279502014-03-27 16:38:30 +0100258 local_bh_disable();
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100259
Daniel Borkmann43279502014-03-27 16:38:30 +0100260 HARD_TX_LOCK(dev, txq, smp_processor_id());
Daniel Borkmann8e2f1a62014-04-02 20:52:57 +0200261 if (!netif_xmit_frozen_or_drv_stopped(txq)) {
David S. Miller47982482014-08-22 16:21:53 -0700262 ret = netdev_start_xmit(skb, dev);
Daniel Borkmann43279502014-03-27 16:38:30 +0100263 if (ret == NETDEV_TX_OK)
264 txq_trans_update(txq);
265 }
266 HARD_TX_UNLOCK(dev, txq);
267
268 local_bh_enable();
269
270 if (!dev_xmit_complete(ret))
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100271 kfree_skb(skb);
Daniel Borkmann43279502014-03-27 16:38:30 +0100272
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100273 return ret;
Daniel Borkmann43279502014-03-27 16:38:30 +0100274drop:
Daniel Borkmann0f97ede2014-04-02 20:52:56 +0200275 atomic_long_inc(&dev->tx_dropped);
Daniel Borkmann43279502014-03-27 16:38:30 +0100276 kfree_skb(skb);
277 return NET_XMIT_DROP;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100278}
279
Daniel Borkmann66e56cd2013-12-06 11:36:15 +0100280static struct net_device *packet_cached_dev_get(struct packet_sock *po)
281{
282 struct net_device *dev;
283
284 rcu_read_lock();
285 dev = rcu_dereference(po->cached_dev);
286 if (likely(dev))
287 dev_hold(dev);
288 rcu_read_unlock();
289
290 return dev;
291}
292
293static void packet_cached_dev_assign(struct packet_sock *po,
294 struct net_device *dev)
295{
296 rcu_assign_pointer(po->cached_dev, dev);
297}
298
299static void packet_cached_dev_reset(struct packet_sock *po)
300{
301 RCU_INIT_POINTER(po->cached_dev, NULL);
302}
303
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100304static bool packet_use_direct_xmit(const struct packet_sock *po)
305{
306 return po->xmit == packet_direct_xmit;
307}
308
Daniel Borkmann0fd5d572014-02-16 15:55:22 +0100309static u16 __packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb)
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100310{
Li Zhong1cbac012013-12-12 22:39:55 +0100311 return (u16) raw_smp_processor_id() % dev->real_num_tx_queues;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100312}
313
Daniel Borkmann0fd5d572014-02-16 15:55:22 +0100314static void packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb)
315{
316 const struct net_device_ops *ops = dev->netdev_ops;
317 u16 queue_index;
318
319 if (ops->ndo_select_queue) {
320 queue_index = ops->ndo_select_queue(dev, skb, NULL,
321 __packet_pick_tx_queue);
322 queue_index = netdev_cap_txqueue(dev, queue_index);
323 } else {
324 queue_index = __packet_pick_tx_queue(dev, skb);
325 }
326
327 skb_set_queue_mapping(skb, queue_index);
328}
329
David S. Millerce06b032011-07-04 01:44:29 -0700330/* register_prot_hook must be invoked with the po->bind_lock held,
331 * or from a context in which asynchronous accesses to the packet
332 * socket is not possible (packet_create()).
333 */
334static void register_prot_hook(struct sock *sk)
335{
336 struct packet_sock *po = pkt_sk(sk);
Daniel Borkmanne40526c2013-11-21 16:50:58 +0100337
David S. Millerce06b032011-07-04 01:44:29 -0700338 if (!po->running) {
Daniel Borkmann66e56cd2013-12-06 11:36:15 +0100339 if (po->fanout)
David S. Millerdc99f602011-07-05 01:45:05 -0700340 __fanout_link(sk, po);
Daniel Borkmann66e56cd2013-12-06 11:36:15 +0100341 else
David S. Millerdc99f602011-07-05 01:45:05 -0700342 dev_add_pack(&po->prot_hook);
Daniel Borkmanne40526c2013-11-21 16:50:58 +0100343
David S. Millerce06b032011-07-04 01:44:29 -0700344 sock_hold(sk);
345 po->running = 1;
346 }
347}
348
349/* {,__}unregister_prot_hook() must be invoked with the po->bind_lock
350 * held. If the sync parameter is true, we will temporarily drop
351 * the po->bind_lock and do a synchronize_net to make sure no
352 * asynchronous packet processing paths still refer to the elements
353 * of po->prot_hook. If the sync parameter is false, it is the
354 * callers responsibility to take care of this.
355 */
356static void __unregister_prot_hook(struct sock *sk, bool sync)
357{
358 struct packet_sock *po = pkt_sk(sk);
359
360 po->running = 0;
Daniel Borkmann66e56cd2013-12-06 11:36:15 +0100361
362 if (po->fanout)
David S. Millerdc99f602011-07-05 01:45:05 -0700363 __fanout_unlink(sk, po);
Daniel Borkmann66e56cd2013-12-06 11:36:15 +0100364 else
David S. Millerdc99f602011-07-05 01:45:05 -0700365 __dev_remove_pack(&po->prot_hook);
Daniel Borkmanne40526c2013-11-21 16:50:58 +0100366
David S. Millerce06b032011-07-04 01:44:29 -0700367 __sock_put(sk);
368
369 if (sync) {
370 spin_unlock(&po->bind_lock);
371 synchronize_net();
372 spin_lock(&po->bind_lock);
373 }
374}
375
376static void unregister_prot_hook(struct sock *sk, bool sync)
377{
378 struct packet_sock *po = pkt_sk(sk);
379
380 if (po->running)
381 __unregister_prot_hook(sk, sync);
382}
383
Changli Gaof6dafa92010-12-07 04:26:16 +0000384static inline __pure struct page *pgv_to_page(void *addr)
Changli Gao0af55bb2010-12-01 02:52:20 +0000385{
386 if (is_vmalloc_addr(addr))
387 return vmalloc_to_page(addr);
388 return virt_to_page(addr);
389}
390
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700391static void __packet_set_status(struct packet_sock *po, void *frame, int status)
392{
Daniel Borkmann184f4892013-04-16 01:57:46 +0000393 union tpacket_uhdr h;
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700394
395 h.raw = frame;
396 switch (po->tp_version) {
397 case TPACKET_V1:
398 h.h1->tp_status = status;
Changli Gao0af55bb2010-12-01 02:52:20 +0000399 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700400 break;
401 case TPACKET_V2:
402 h.h2->tp_status = status;
Changli Gao0af55bb2010-12-01 02:52:20 +0000403 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700404 break;
chetan lokef6fb8f102011-08-19 10:18:16 +0000405 case TPACKET_V3:
Johann Baudy69e3c752009-05-18 22:11:22 -0700406 default:
chetan lokef6fb8f102011-08-19 10:18:16 +0000407 WARN(1, "TPACKET version not supported.\n");
Johann Baudy69e3c752009-05-18 22:11:22 -0700408 BUG();
409 }
410
411 smp_wmb();
412}
413
414static int __packet_get_status(struct packet_sock *po, void *frame)
415{
Daniel Borkmann184f4892013-04-16 01:57:46 +0000416 union tpacket_uhdr h;
Johann Baudy69e3c752009-05-18 22:11:22 -0700417
418 smp_rmb();
419
420 h.raw = frame;
421 switch (po->tp_version) {
422 case TPACKET_V1:
Changli Gao0af55bb2010-12-01 02:52:20 +0000423 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
Johann Baudy69e3c752009-05-18 22:11:22 -0700424 return h.h1->tp_status;
425 case TPACKET_V2:
Changli Gao0af55bb2010-12-01 02:52:20 +0000426 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
Johann Baudy69e3c752009-05-18 22:11:22 -0700427 return h.h2->tp_status;
chetan lokef6fb8f102011-08-19 10:18:16 +0000428 case TPACKET_V3:
Johann Baudy69e3c752009-05-18 22:11:22 -0700429 default:
chetan lokef6fb8f102011-08-19 10:18:16 +0000430 WARN(1, "TPACKET version not supported.\n");
Johann Baudy69e3c752009-05-18 22:11:22 -0700431 BUG();
432 return 0;
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700433 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700434}
Johann Baudy69e3c752009-05-18 22:11:22 -0700435
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000436static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec *ts,
437 unsigned int flags)
Daniel Borkmann7a513842013-04-23 00:39:29 +0000438{
439 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
440
Willem de Bruijn68a360e2014-07-25 18:01:31 -0400441 if (shhwtstamps &&
442 (flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
443 ktime_to_timespec_cond(shhwtstamps->hwtstamp, ts))
444 return TP_STATUS_TS_RAW_HARDWARE;
Daniel Borkmann7a513842013-04-23 00:39:29 +0000445
446 if (ktime_to_timespec_cond(skb->tstamp, ts))
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000447 return TP_STATUS_TS_SOFTWARE;
Daniel Borkmann7a513842013-04-23 00:39:29 +0000448
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000449 return 0;
Daniel Borkmann7a513842013-04-23 00:39:29 +0000450}
451
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000452static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
453 struct sk_buff *skb)
Willem de Bruijn2e313962013-04-23 00:39:28 +0000454{
455 union tpacket_uhdr h;
456 struct timespec ts;
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000457 __u32 ts_status;
Willem de Bruijn2e313962013-04-23 00:39:28 +0000458
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000459 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
460 return 0;
Willem de Bruijn2e313962013-04-23 00:39:28 +0000461
462 h.raw = frame;
463 switch (po->tp_version) {
464 case TPACKET_V1:
465 h.h1->tp_sec = ts.tv_sec;
466 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
467 break;
468 case TPACKET_V2:
469 h.h2->tp_sec = ts.tv_sec;
470 h.h2->tp_nsec = ts.tv_nsec;
471 break;
472 case TPACKET_V3:
473 default:
474 WARN(1, "TPACKET version not supported.\n");
475 BUG();
476 }
477
478 /* one flush is safe, as both fields always lie on the same cacheline */
479 flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
480 smp_wmb();
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000481
482 return ts_status;
Willem de Bruijn2e313962013-04-23 00:39:28 +0000483}
484
Johann Baudy69e3c752009-05-18 22:11:22 -0700485static void *packet_lookup_frame(struct packet_sock *po,
486 struct packet_ring_buffer *rb,
487 unsigned int position,
488 int status)
489{
490 unsigned int pg_vec_pos, frame_offset;
Daniel Borkmann184f4892013-04-16 01:57:46 +0000491 union tpacket_uhdr h;
Johann Baudy69e3c752009-05-18 22:11:22 -0700492
493 pg_vec_pos = position / rb->frames_per_block;
494 frame_offset = position % rb->frames_per_block;
495
Neil Horman0e3125c2010-11-16 10:26:47 -0800496 h.raw = rb->pg_vec[pg_vec_pos].buffer +
497 (frame_offset * rb->frame_size);
Johann Baudy69e3c752009-05-18 22:11:22 -0700498
499 if (status != __packet_get_status(po, h.raw))
500 return NULL;
501
502 return h.raw;
503}
504
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000505static void *packet_current_frame(struct packet_sock *po,
Johann Baudy69e3c752009-05-18 22:11:22 -0700506 struct packet_ring_buffer *rb,
507 int status)
508{
509 return packet_lookup_frame(po, rb, rb->head, status);
510}
511
chetan lokebc59ba32011-08-25 10:43:30 +0000512static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
chetan lokef6fb8f102011-08-19 10:18:16 +0000513{
514 del_timer_sync(&pkc->retire_blk_timer);
515}
516
517static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
518 int tx_ring,
519 struct sk_buff_head *rb_queue)
520{
chetan lokebc59ba32011-08-25 10:43:30 +0000521 struct tpacket_kbdq_core *pkc;
chetan lokef6fb8f102011-08-19 10:18:16 +0000522
Duan Jiong22781a52013-12-06 13:29:36 +0800523 pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :
524 GET_PBDQC_FROM_RB(&po->rx_ring);
chetan lokef6fb8f102011-08-19 10:18:16 +0000525
Veaceslav Falicoec6f8092013-11-29 09:53:23 +0100526 spin_lock_bh(&rb_queue->lock);
chetan lokef6fb8f102011-08-19 10:18:16 +0000527 pkc->delete_blk_timer = 1;
Veaceslav Falicoec6f8092013-11-29 09:53:23 +0100528 spin_unlock_bh(&rb_queue->lock);
chetan lokef6fb8f102011-08-19 10:18:16 +0000529
530 prb_del_retire_blk_timer(pkc);
531}
532
533static void prb_init_blk_timer(struct packet_sock *po,
chetan lokebc59ba32011-08-25 10:43:30 +0000534 struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f102011-08-19 10:18:16 +0000535 void (*func) (unsigned long))
536{
537 init_timer(&pkc->retire_blk_timer);
538 pkc->retire_blk_timer.data = (long)po;
539 pkc->retire_blk_timer.function = func;
540 pkc->retire_blk_timer.expires = jiffies;
541}
542
543static void prb_setup_retire_blk_timer(struct packet_sock *po, int tx_ring)
544{
chetan lokebc59ba32011-08-25 10:43:30 +0000545 struct tpacket_kbdq_core *pkc;
chetan lokef6fb8f102011-08-19 10:18:16 +0000546
547 if (tx_ring)
548 BUG();
549
Duan Jiong22781a52013-12-06 13:29:36 +0800550 pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :
551 GET_PBDQC_FROM_RB(&po->rx_ring);
chetan lokef6fb8f102011-08-19 10:18:16 +0000552 prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);
553}
554
555static int prb_calc_retire_blk_tmo(struct packet_sock *po,
556 int blk_size_in_bytes)
557{
558 struct net_device *dev;
559 unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
Jiri Pirko4bc71cb2011-09-03 03:34:30 +0000560 struct ethtool_cmd ecmd;
561 int err;
parav.pandit@emulex.come440cf22012-06-27 03:56:12 +0000562 u32 speed;
chetan lokef6fb8f102011-08-19 10:18:16 +0000563
Jiri Pirko4bc71cb2011-09-03 03:34:30 +0000564 rtnl_lock();
565 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
566 if (unlikely(!dev)) {
567 rtnl_unlock();
chetan lokef6fb8f102011-08-19 10:18:16 +0000568 return DEFAULT_PRB_RETIRE_TOV;
Jiri Pirko4bc71cb2011-09-03 03:34:30 +0000569 }
570 err = __ethtool_get_settings(dev, &ecmd);
parav.pandit@emulex.come440cf22012-06-27 03:56:12 +0000571 speed = ethtool_cmd_speed(&ecmd);
Jiri Pirko4bc71cb2011-09-03 03:34:30 +0000572 rtnl_unlock();
573 if (!err) {
Jiri Pirko4bc71cb2011-09-03 03:34:30 +0000574 /*
575 * If the link speed is so slow you don't really
576 * need to worry about perf anyways
577 */
parav.pandit@emulex.come440cf22012-06-27 03:56:12 +0000578 if (speed < SPEED_1000 || speed == SPEED_UNKNOWN) {
Jiri Pirko4bc71cb2011-09-03 03:34:30 +0000579 return DEFAULT_PRB_RETIRE_TOV;
parav.pandit@emulex.come440cf22012-06-27 03:56:12 +0000580 } else {
581 msec = 1;
582 div = speed / 1000;
chetan lokef6fb8f102011-08-19 10:18:16 +0000583 }
584 }
585
586 mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
587
588 if (div)
589 mbits /= div;
590
591 tmo = mbits * msec;
592
593 if (div)
594 return tmo+1;
595 return tmo;
596}
597
chetan lokebc59ba32011-08-25 10:43:30 +0000598static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
chetan lokef6fb8f102011-08-19 10:18:16 +0000599 union tpacket_req_u *req_u)
600{
601 p1->feature_req_word = req_u->req3.tp_feature_req_word;
602}
603
604static void init_prb_bdqc(struct packet_sock *po,
605 struct packet_ring_buffer *rb,
606 struct pgv *pg_vec,
607 union tpacket_req_u *req_u, int tx_ring)
608{
Duan Jiong22781a52013-12-06 13:29:36 +0800609 struct tpacket_kbdq_core *p1 = GET_PBDQC_FROM_RB(rb);
chetan lokebc59ba32011-08-25 10:43:30 +0000610 struct tpacket_block_desc *pbd;
chetan lokef6fb8f102011-08-19 10:18:16 +0000611
612 memset(p1, 0x0, sizeof(*p1));
613
614 p1->knxt_seq_num = 1;
615 p1->pkbdq = pg_vec;
chetan lokebc59ba32011-08-25 10:43:30 +0000616 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
Joe Perchese3192692012-06-03 17:41:40 +0000617 p1->pkblk_start = pg_vec[0].buffer;
chetan lokef6fb8f102011-08-19 10:18:16 +0000618 p1->kblk_size = req_u->req3.tp_block_size;
619 p1->knum_blocks = req_u->req3.tp_block_nr;
620 p1->hdrlen = po->tp_hdrlen;
621 p1->version = po->tp_version;
622 p1->last_kactive_blk_num = 0;
Daniel Borkmannee80fbf2013-04-19 06:12:29 +0000623 po->stats.stats3.tp_freeze_q_cnt = 0;
chetan lokef6fb8f102011-08-19 10:18:16 +0000624 if (req_u->req3.tp_retire_blk_tov)
625 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
626 else
627 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
628 req_u->req3.tp_block_size);
629 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
630 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
631
Eric Dumazetdc808112014-08-15 09:16:04 -0700632 p1->max_frame_len = p1->kblk_size - BLK_PLUS_PRIV(p1->blk_sizeof_priv);
chetan lokef6fb8f102011-08-19 10:18:16 +0000633 prb_init_ft_ops(p1, req_u);
634 prb_setup_retire_blk_timer(po, tx_ring);
635 prb_open_block(p1, pbd);
636}
637
638/* Do NOT update the last_blk_num first.
639 * Assumes sk_buff_head lock is held.
640 */
chetan lokebc59ba32011-08-25 10:43:30 +0000641static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
chetan lokef6fb8f102011-08-19 10:18:16 +0000642{
643 mod_timer(&pkc->retire_blk_timer,
644 jiffies + pkc->tov_in_jiffies);
645 pkc->last_kactive_blk_num = pkc->kactive_blk_num;
646}
647
648/*
649 * Timer logic:
650 * 1) We refresh the timer only when we open a block.
651 * By doing this we don't waste cycles refreshing the timer
652 * on packet-by-packet basis.
653 *
654 * With a 1MB block-size, on a 1Gbps line, it will take
655 * i) ~8 ms to fill a block + ii) memcpy etc.
656 * In this cut we are not accounting for the memcpy time.
657 *
658 * So, if the user sets the 'tmo' to 10ms then the timer
659 * will never fire while the block is still getting filled
660 * (which is what we want). However, the user could choose
661 * to close a block early and that's fine.
662 *
663 * But when the timer does fire, we check whether or not to refresh it.
664 * Since the tmo granularity is in msecs, it is not too expensive
665 * to refresh the timer, lets say every '8' msecs.
666 * Either the user can set the 'tmo' or we can derive it based on
667 * a) line-speed and b) block-size.
668 * prb_calc_retire_blk_tmo() calculates the tmo.
669 *
670 */
671static void prb_retire_rx_blk_timer_expired(unsigned long data)
672{
673 struct packet_sock *po = (struct packet_sock *)data;
Duan Jiong22781a52013-12-06 13:29:36 +0800674 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
chetan lokef6fb8f102011-08-19 10:18:16 +0000675 unsigned int frozen;
chetan lokebc59ba32011-08-25 10:43:30 +0000676 struct tpacket_block_desc *pbd;
chetan lokef6fb8f102011-08-19 10:18:16 +0000677
678 spin_lock(&po->sk.sk_receive_queue.lock);
679
680 frozen = prb_queue_frozen(pkc);
681 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
682
683 if (unlikely(pkc->delete_blk_timer))
684 goto out;
685
686 /* We only need to plug the race when the block is partially filled.
687 * tpacket_rcv:
688 * lock(); increment BLOCK_NUM_PKTS; unlock()
689 * copy_bits() is in progress ...
690 * timer fires on other cpu:
691 * we can't retire the current block because copy_bits
692 * is in progress.
693 *
694 */
695 if (BLOCK_NUM_PKTS(pbd)) {
696 while (atomic_read(&pkc->blk_fill_in_prog)) {
697 /* Waiting for skb_copy_bits to finish... */
698 cpu_relax();
699 }
700 }
701
702 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
703 if (!frozen) {
704 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
705 if (!prb_dispatch_next_block(pkc, po))
706 goto refresh_timer;
707 else
708 goto out;
709 } else {
710 /* Case 1. Queue was frozen because user-space was
711 * lagging behind.
712 */
713 if (prb_curr_blk_in_use(pkc, pbd)) {
714 /*
715 * Ok, user-space is still behind.
716 * So just refresh the timer.
717 */
718 goto refresh_timer;
719 } else {
720 /* Case 2. queue was frozen,user-space caught up,
721 * now the link went idle && the timer fired.
722 * We don't have a block to close.So we open this
723 * block and restart the timer.
724 * opening a block thaws the queue,restarts timer
725 * Thawing/timer-refresh is a side effect.
726 */
727 prb_open_block(pkc, pbd);
728 goto out;
729 }
730 }
731 }
732
733refresh_timer:
734 _prb_refresh_rx_retire_blk_timer(pkc);
735
736out:
737 spin_unlock(&po->sk.sk_receive_queue.lock);
738}
739
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000740static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
chetan lokebc59ba32011-08-25 10:43:30 +0000741 struct tpacket_block_desc *pbd1, __u32 status)
chetan lokef6fb8f102011-08-19 10:18:16 +0000742{
743 /* Flush everything minus the block header */
744
745#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
746 u8 *start, *end;
747
748 start = (u8 *)pbd1;
749
750 /* Skip the block header(we know header WILL fit in 4K) */
751 start += PAGE_SIZE;
752
753 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
754 for (; start < end; start += PAGE_SIZE)
755 flush_dcache_page(pgv_to_page(start));
756
757 smp_wmb();
758#endif
759
760 /* Now update the block status. */
761
762 BLOCK_STATUS(pbd1) = status;
763
764 /* Flush the block header */
765
766#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
767 start = (u8 *)pbd1;
768 flush_dcache_page(pgv_to_page(start));
769
770 smp_wmb();
771#endif
772}
773
774/*
775 * Side effect:
776 *
777 * 1) flush the block
778 * 2) Increment active_blk_num
779 *
780 * Note:We DONT refresh the timer on purpose.
781 * Because almost always the next block will be opened.
782 */
chetan lokebc59ba32011-08-25 10:43:30 +0000783static void prb_close_block(struct tpacket_kbdq_core *pkc1,
784 struct tpacket_block_desc *pbd1,
chetan lokef6fb8f102011-08-19 10:18:16 +0000785 struct packet_sock *po, unsigned int stat)
786{
787 __u32 status = TP_STATUS_USER | stat;
788
789 struct tpacket3_hdr *last_pkt;
chetan lokebc59ba32011-08-25 10:43:30 +0000790 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
chetan lokef6fb8f102011-08-19 10:18:16 +0000791
Daniel Borkmannee80fbf2013-04-19 06:12:29 +0000792 if (po->stats.stats3.tp_drops)
chetan lokef6fb8f102011-08-19 10:18:16 +0000793 status |= TP_STATUS_LOSING;
794
795 last_pkt = (struct tpacket3_hdr *)pkc1->prev;
796 last_pkt->tp_next_offset = 0;
797
798 /* Get the ts of the last pkt */
799 if (BLOCK_NUM_PKTS(pbd1)) {
800 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
801 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
802 } else {
803 /* Ok, we tmo'd - so get the current time */
804 struct timespec ts;
805 getnstimeofday(&ts);
806 h1->ts_last_pkt.ts_sec = ts.tv_sec;
807 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
808 }
809
810 smp_wmb();
811
812 /* Flush the block */
813 prb_flush_block(pkc1, pbd1, status);
814
815 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
816}
817
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000818static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
chetan lokef6fb8f102011-08-19 10:18:16 +0000819{
820 pkc->reset_pending_on_curr_blk = 0;
821}
822
823/*
824 * Side effect of opening a block:
825 *
826 * 1) prb_queue is thawed.
827 * 2) retire_blk_timer is refreshed.
828 *
829 */
chetan lokebc59ba32011-08-25 10:43:30 +0000830static void prb_open_block(struct tpacket_kbdq_core *pkc1,
831 struct tpacket_block_desc *pbd1)
chetan lokef6fb8f102011-08-19 10:18:16 +0000832{
833 struct timespec ts;
chetan lokebc59ba32011-08-25 10:43:30 +0000834 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
chetan lokef6fb8f102011-08-19 10:18:16 +0000835
836 smp_rmb();
837
Daniel Borkmann8da30562013-05-03 02:57:00 +0000838 /* We could have just memset this but we will lose the
839 * flexibility of making the priv area sticky
840 */
chetan lokef6fb8f102011-08-19 10:18:16 +0000841
Daniel Borkmann8da30562013-05-03 02:57:00 +0000842 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
843 BLOCK_NUM_PKTS(pbd1) = 0;
844 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
chetan lokef6fb8f102011-08-19 10:18:16 +0000845
Daniel Borkmann8da30562013-05-03 02:57:00 +0000846 getnstimeofday(&ts);
chetan lokef6fb8f102011-08-19 10:18:16 +0000847
Daniel Borkmann8da30562013-05-03 02:57:00 +0000848 h1->ts_first_pkt.ts_sec = ts.tv_sec;
849 h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
chetan lokef6fb8f102011-08-19 10:18:16 +0000850
Daniel Borkmann8da30562013-05-03 02:57:00 +0000851 pkc1->pkblk_start = (char *)pbd1;
852 pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
853
854 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
855 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
856
857 pbd1->version = pkc1->version;
858 pkc1->prev = pkc1->nxt_offset;
859 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
860
861 prb_thaw_queue(pkc1);
862 _prb_refresh_rx_retire_blk_timer(pkc1);
863
864 smp_wmb();
chetan lokef6fb8f102011-08-19 10:18:16 +0000865}
866
867/*
868 * Queue freeze logic:
869 * 1) Assume tp_block_nr = 8 blocks.
870 * 2) At time 't0', user opens Rx ring.
871 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
872 * 4) user-space is either sleeping or processing block '0'.
873 * 5) tpacket_rcv is currently filling block '7', since there is no space left,
874 * it will close block-7,loop around and try to fill block '0'.
875 * call-flow:
876 * __packet_lookup_frame_in_block
877 * prb_retire_current_block()
878 * prb_dispatch_next_block()
879 * |->(BLOCK_STATUS == USER) evaluates to true
880 * 5.1) Since block-0 is currently in-use, we just freeze the queue.
881 * 6) Now there are two cases:
882 * 6.1) Link goes idle right after the queue is frozen.
883 * But remember, the last open_block() refreshed the timer.
884 * When this timer expires,it will refresh itself so that we can
885 * re-open block-0 in near future.
886 * 6.2) Link is busy and keeps on receiving packets. This is a simple
887 * case and __packet_lookup_frame_in_block will check if block-0
888 * is free and can now be re-used.
889 */
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000890static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f102011-08-19 10:18:16 +0000891 struct packet_sock *po)
892{
893 pkc->reset_pending_on_curr_blk = 1;
Daniel Borkmannee80fbf2013-04-19 06:12:29 +0000894 po->stats.stats3.tp_freeze_q_cnt++;
chetan lokef6fb8f102011-08-19 10:18:16 +0000895}
896
897#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
898
899/*
900 * If the next block is free then we will dispatch it
901 * and return a good offset.
902 * Else, we will freeze the queue.
903 * So, caller must check the return value.
904 */
chetan lokebc59ba32011-08-25 10:43:30 +0000905static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f102011-08-19 10:18:16 +0000906 struct packet_sock *po)
907{
chetan lokebc59ba32011-08-25 10:43:30 +0000908 struct tpacket_block_desc *pbd;
chetan lokef6fb8f102011-08-19 10:18:16 +0000909
910 smp_rmb();
911
912 /* 1. Get current block num */
913 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
914
915 /* 2. If this block is currently in_use then freeze the queue */
916 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
917 prb_freeze_queue(pkc, po);
918 return NULL;
919 }
920
921 /*
922 * 3.
923 * open this block and return the offset where the first packet
924 * needs to get stored.
925 */
926 prb_open_block(pkc, pbd);
927 return (void *)pkc->nxt_offset;
928}
929
chetan lokebc59ba32011-08-25 10:43:30 +0000930static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f102011-08-19 10:18:16 +0000931 struct packet_sock *po, unsigned int status)
932{
chetan lokebc59ba32011-08-25 10:43:30 +0000933 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
chetan lokef6fb8f102011-08-19 10:18:16 +0000934
935 /* retire/close the current block */
936 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
937 /*
938 * Plug the case where copy_bits() is in progress on
939 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
940 * have space to copy the pkt in the current block and
941 * called prb_retire_current_block()
942 *
943 * We don't need to worry about the TMO case because
944 * the timer-handler already handled this case.
945 */
946 if (!(status & TP_STATUS_BLK_TMO)) {
947 while (atomic_read(&pkc->blk_fill_in_prog)) {
948 /* Waiting for skb_copy_bits to finish... */
949 cpu_relax();
950 }
951 }
952 prb_close_block(pkc, pbd, po, status);
953 return;
954 }
chetan lokef6fb8f102011-08-19 10:18:16 +0000955}
956
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000957static int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,
chetan lokebc59ba32011-08-25 10:43:30 +0000958 struct tpacket_block_desc *pbd)
chetan lokef6fb8f102011-08-19 10:18:16 +0000959{
960 return TP_STATUS_USER & BLOCK_STATUS(pbd);
961}
962
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000963static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
chetan lokef6fb8f102011-08-19 10:18:16 +0000964{
965 return pkc->reset_pending_on_curr_blk;
966}
967
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000968static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
chetan lokef6fb8f102011-08-19 10:18:16 +0000969{
chetan lokebc59ba32011-08-25 10:43:30 +0000970 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
chetan lokef6fb8f102011-08-19 10:18:16 +0000971 atomic_dec(&pkc->blk_fill_in_prog);
972}
973
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000974static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f102011-08-19 10:18:16 +0000975 struct tpacket3_hdr *ppd)
976{
Tom Herbert3958afa1b2013-12-15 22:12:06 -0800977 ppd->hv1.tp_rxhash = skb_get_hash(pkc->skb);
chetan lokef6fb8f102011-08-19 10:18:16 +0000978}
979
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000980static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f102011-08-19 10:18:16 +0000981 struct tpacket3_hdr *ppd)
982{
983 ppd->hv1.tp_rxhash = 0;
984}
985
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000986static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f102011-08-19 10:18:16 +0000987 struct tpacket3_hdr *ppd)
988{
989 if (vlan_tx_tag_present(pkc->skb)) {
990 ppd->hv1.tp_vlan_tci = vlan_tx_tag_get(pkc->skb);
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +0900991 ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->vlan_proto);
992 ppd->tp_status = TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
chetan lokef6fb8f102011-08-19 10:18:16 +0000993 } else {
danborkmann@iogearbox.net9e670302012-08-20 03:34:03 +0000994 ppd->hv1.tp_vlan_tci = 0;
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +0900995 ppd->hv1.tp_vlan_tpid = 0;
danborkmann@iogearbox.net9e670302012-08-20 03:34:03 +0000996 ppd->tp_status = TP_STATUS_AVAILABLE;
chetan lokef6fb8f102011-08-19 10:18:16 +0000997 }
998}
999
chetan lokebc59ba32011-08-25 10:43:30 +00001000static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f102011-08-19 10:18:16 +00001001 struct tpacket3_hdr *ppd)
1002{
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +09001003 ppd->hv1.tp_padding = 0;
chetan lokef6fb8f102011-08-19 10:18:16 +00001004 prb_fill_vlan_info(pkc, ppd);
1005
1006 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
1007 prb_fill_rxhash(pkc, ppd);
1008 else
1009 prb_clear_rxhash(pkc, ppd);
1010}
1011
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001012static void prb_fill_curr_block(char *curr,
chetan lokebc59ba32011-08-25 10:43:30 +00001013 struct tpacket_kbdq_core *pkc,
1014 struct tpacket_block_desc *pbd,
chetan lokef6fb8f102011-08-19 10:18:16 +00001015 unsigned int len)
1016{
1017 struct tpacket3_hdr *ppd;
1018
1019 ppd = (struct tpacket3_hdr *)curr;
1020 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
1021 pkc->prev = curr;
1022 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
1023 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
1024 BLOCK_NUM_PKTS(pbd) += 1;
1025 atomic_inc(&pkc->blk_fill_in_prog);
1026 prb_run_all_ft_ops(pkc, ppd);
1027}
1028
1029/* Assumes caller has the sk->rx_queue.lock */
1030static void *__packet_lookup_frame_in_block(struct packet_sock *po,
1031 struct sk_buff *skb,
1032 int status,
1033 unsigned int len
1034 )
1035{
chetan lokebc59ba32011-08-25 10:43:30 +00001036 struct tpacket_kbdq_core *pkc;
1037 struct tpacket_block_desc *pbd;
chetan lokef6fb8f102011-08-19 10:18:16 +00001038 char *curr, *end;
1039
Joe Perchese3192692012-06-03 17:41:40 +00001040 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
chetan lokef6fb8f102011-08-19 10:18:16 +00001041 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1042
1043 /* Queue is frozen when user space is lagging behind */
1044 if (prb_queue_frozen(pkc)) {
1045 /*
1046 * Check if that last block which caused the queue to freeze,
1047 * is still in_use by user-space.
1048 */
1049 if (prb_curr_blk_in_use(pkc, pbd)) {
1050 /* Can't record this packet */
1051 return NULL;
1052 } else {
1053 /*
1054 * Ok, the block was released by user-space.
1055 * Now let's open that block.
1056 * opening a block also thaws the queue.
1057 * Thawing is a side effect.
1058 */
1059 prb_open_block(pkc, pbd);
1060 }
1061 }
1062
1063 smp_mb();
1064 curr = pkc->nxt_offset;
1065 pkc->skb = skb;
Joe Perchese3192692012-06-03 17:41:40 +00001066 end = (char *)pbd + pkc->kblk_size;
chetan lokef6fb8f102011-08-19 10:18:16 +00001067
1068 /* first try the current block */
1069 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
1070 prb_fill_curr_block(curr, pkc, pbd, len);
1071 return (void *)curr;
1072 }
1073
1074 /* Ok, close the current block */
1075 prb_retire_current_block(pkc, po, 0);
1076
1077 /* Now, try to dispatch the next block */
1078 curr = (char *)prb_dispatch_next_block(pkc, po);
1079 if (curr) {
1080 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1081 prb_fill_curr_block(curr, pkc, pbd, len);
1082 return (void *)curr;
1083 }
1084
1085 /*
1086 * No free blocks are available.user_space hasn't caught up yet.
1087 * Queue was just frozen and now this packet will get dropped.
1088 */
1089 return NULL;
1090}
1091
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001092static void *packet_current_rx_frame(struct packet_sock *po,
chetan lokef6fb8f102011-08-19 10:18:16 +00001093 struct sk_buff *skb,
1094 int status, unsigned int len)
1095{
1096 char *curr = NULL;
1097 switch (po->tp_version) {
1098 case TPACKET_V1:
1099 case TPACKET_V2:
1100 curr = packet_lookup_frame(po, &po->rx_ring,
1101 po->rx_ring.head, status);
1102 return curr;
1103 case TPACKET_V3:
1104 return __packet_lookup_frame_in_block(po, skb, status, len);
1105 default:
1106 WARN(1, "TPACKET version not supported\n");
1107 BUG();
Ying Xue99aa3472012-08-06 16:27:10 +00001108 return NULL;
chetan lokef6fb8f102011-08-19 10:18:16 +00001109 }
1110}
1111
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001112static void *prb_lookup_block(struct packet_sock *po,
chetan lokef6fb8f102011-08-19 10:18:16 +00001113 struct packet_ring_buffer *rb,
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001114 unsigned int idx,
chetan lokef6fb8f102011-08-19 10:18:16 +00001115 int status)
1116{
chetan lokebc59ba32011-08-25 10:43:30 +00001117 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001118 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
chetan lokef6fb8f102011-08-19 10:18:16 +00001119
1120 if (status != BLOCK_STATUS(pbd))
1121 return NULL;
1122 return pbd;
1123}
1124
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001125static int prb_previous_blk_num(struct packet_ring_buffer *rb)
chetan lokef6fb8f102011-08-19 10:18:16 +00001126{
1127 unsigned int prev;
1128 if (rb->prb_bdqc.kactive_blk_num)
1129 prev = rb->prb_bdqc.kactive_blk_num-1;
1130 else
1131 prev = rb->prb_bdqc.knum_blocks-1;
1132 return prev;
1133}
1134
1135/* Assumes caller has held the rx_queue.lock */
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001136static void *__prb_previous_block(struct packet_sock *po,
chetan lokef6fb8f102011-08-19 10:18:16 +00001137 struct packet_ring_buffer *rb,
1138 int status)
1139{
1140 unsigned int previous = prb_previous_blk_num(rb);
1141 return prb_lookup_block(po, rb, previous, status);
1142}
1143
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001144static void *packet_previous_rx_frame(struct packet_sock *po,
chetan lokef6fb8f102011-08-19 10:18:16 +00001145 struct packet_ring_buffer *rb,
1146 int status)
1147{
1148 if (po->tp_version <= TPACKET_V2)
1149 return packet_previous_frame(po, rb, status);
1150
1151 return __prb_previous_block(po, rb, status);
1152}
1153
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001154static void packet_increment_rx_head(struct packet_sock *po,
chetan lokef6fb8f102011-08-19 10:18:16 +00001155 struct packet_ring_buffer *rb)
1156{
1157 switch (po->tp_version) {
1158 case TPACKET_V1:
1159 case TPACKET_V2:
1160 return packet_increment_head(rb);
1161 case TPACKET_V3:
1162 default:
1163 WARN(1, "TPACKET version not supported.\n");
1164 BUG();
1165 return;
1166 }
1167}
1168
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001169static void *packet_previous_frame(struct packet_sock *po,
Johann Baudy69e3c752009-05-18 22:11:22 -07001170 struct packet_ring_buffer *rb,
1171 int status)
1172{
1173 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1174 return packet_lookup_frame(po, rb, previous, status);
1175}
1176
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001177static void packet_increment_head(struct packet_ring_buffer *buff)
Johann Baudy69e3c752009-05-18 22:11:22 -07001178{
1179 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1180}
1181
Daniel Borkmannb0138402014-01-15 16:25:36 +01001182static void packet_inc_pending(struct packet_ring_buffer *rb)
1183{
1184 this_cpu_inc(*rb->pending_refcnt);
1185}
1186
1187static void packet_dec_pending(struct packet_ring_buffer *rb)
1188{
1189 this_cpu_dec(*rb->pending_refcnt);
1190}
1191
1192static unsigned int packet_read_pending(const struct packet_ring_buffer *rb)
1193{
1194 unsigned int refcnt = 0;
1195 int cpu;
1196
1197 /* We don't use pending refcount in rx_ring. */
1198 if (rb->pending_refcnt == NULL)
1199 return 0;
1200
1201 for_each_possible_cpu(cpu)
1202 refcnt += *per_cpu_ptr(rb->pending_refcnt, cpu);
1203
1204 return refcnt;
1205}
1206
1207static int packet_alloc_pending(struct packet_sock *po)
1208{
1209 po->rx_ring.pending_refcnt = NULL;
1210
1211 po->tx_ring.pending_refcnt = alloc_percpu(unsigned int);
1212 if (unlikely(po->tx_ring.pending_refcnt == NULL))
1213 return -ENOBUFS;
1214
1215 return 0;
1216}
1217
1218static void packet_free_pending(struct packet_sock *po)
1219{
1220 free_percpu(po->tx_ring.pending_refcnt);
1221}
1222
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001223static bool packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1224{
1225 struct sock *sk = &po->sk;
1226 bool has_room;
1227
1228 if (po->prot_hook.func != tpacket_rcv)
1229 return (atomic_read(&sk->sk_rmem_alloc) + skb->truesize)
1230 <= sk->sk_rcvbuf;
1231
1232 spin_lock(&sk->sk_receive_queue.lock);
1233 if (po->tp_version == TPACKET_V3)
1234 has_room = prb_lookup_block(po, &po->rx_ring,
1235 po->rx_ring.prb_bdqc.kactive_blk_num,
1236 TP_STATUS_KERNEL);
1237 else
1238 has_room = packet_lookup_frame(po, &po->rx_ring,
1239 po->rx_ring.head,
1240 TP_STATUS_KERNEL);
1241 spin_unlock(&sk->sk_receive_queue.lock);
1242
1243 return has_room;
1244}
1245
Linus Torvalds1da177e2005-04-16 15:20:36 -07001246static void packet_sock_destruct(struct sock *sk)
1247{
Richard Cochraned85b562010-04-07 22:41:28 +00001248 skb_queue_purge(&sk->sk_error_queue);
1249
Ilpo Järvinen547b7922008-07-25 21:43:18 -07001250 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1251 WARN_ON(atomic_read(&sk->sk_wmem_alloc));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001252
1253 if (!sock_flag(sk, SOCK_DEAD)) {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001254 pr_err("Attempt to release alive packet socket: %p\n", sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001255 return;
1256 }
1257
Pavel Emelyanov17ab56a2007-11-10 21:38:48 -08001258 sk_refcnt_debug_dec(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001259}
1260
David S. Millerdc99f602011-07-05 01:45:05 -07001261static int fanout_rr_next(struct packet_fanout *f, unsigned int num)
1262{
1263 int x = atomic_read(&f->rr_cur) + 1;
1264
1265 if (x >= num)
1266 x = 0;
1267
1268 return x;
1269}
1270
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001271static unsigned int fanout_demux_hash(struct packet_fanout *f,
1272 struct sk_buff *skb,
1273 unsigned int num)
David S. Millerdc99f602011-07-05 01:45:05 -07001274{
Tom Herbert61b905d2014-03-24 15:34:47 -07001275 return reciprocal_scale(skb_get_hash(skb), num);
David S. Millerdc99f602011-07-05 01:45:05 -07001276}
1277
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001278static unsigned int fanout_demux_lb(struct packet_fanout *f,
1279 struct sk_buff *skb,
1280 unsigned int num)
David S. Millerdc99f602011-07-05 01:45:05 -07001281{
1282 int cur, old;
1283
1284 cur = atomic_read(&f->rr_cur);
1285 while ((old = atomic_cmpxchg(&f->rr_cur, cur,
1286 fanout_rr_next(f, num))) != cur)
1287 cur = old;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001288 return cur;
David S. Millerdc99f602011-07-05 01:45:05 -07001289}
1290
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001291static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1292 struct sk_buff *skb,
1293 unsigned int num)
David S. Miller95ec3eb2011-07-06 01:56:38 -07001294{
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001295 return smp_processor_id() % num;
1296}
David S. Miller95ec3eb2011-07-06 01:56:38 -07001297
Daniel Borkmann5df0ddf2013-08-28 22:13:09 +02001298static unsigned int fanout_demux_rnd(struct packet_fanout *f,
1299 struct sk_buff *skb,
1300 unsigned int num)
1301{
Daniel Borkmannf337db62014-01-22 02:29:39 +01001302 return prandom_u32_max(num);
Daniel Borkmann5df0ddf2013-08-28 22:13:09 +02001303}
1304
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001305static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1306 struct sk_buff *skb,
1307 unsigned int idx, unsigned int skip,
1308 unsigned int num)
1309{
1310 unsigned int i, j;
1311
1312 i = j = min_t(int, f->next[idx], num - 1);
1313 do {
1314 if (i != skip && packet_rcv_has_room(pkt_sk(f->arr[i]), skb)) {
1315 if (i != j)
1316 f->next[idx] = i;
1317 return i;
1318 }
1319 if (++i == num)
1320 i = 0;
1321 } while (i != j);
1322
1323 return idx;
1324}
1325
Neil Horman2d360972014-01-22 16:01:44 -05001326static unsigned int fanout_demux_qm(struct packet_fanout *f,
1327 struct sk_buff *skb,
1328 unsigned int num)
1329{
1330 return skb_get_queue_mapping(skb) % num;
1331}
1332
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001333static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1334{
1335 return f->flags & (flag >> 8);
David S. Miller95ec3eb2011-07-06 01:56:38 -07001336}
1337
David S. Miller95ec3eb2011-07-06 01:56:38 -07001338static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1339 struct packet_type *pt, struct net_device *orig_dev)
David S. Millerdc99f602011-07-05 01:45:05 -07001340{
1341 struct packet_fanout *f = pt->af_packet_priv;
1342 unsigned int num = f->num_members;
1343 struct packet_sock *po;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001344 unsigned int idx;
David S. Millerdc99f602011-07-05 01:45:05 -07001345
1346 if (!net_eq(dev_net(dev), read_pnet(&f->net)) ||
1347 !num) {
1348 kfree_skb(skb);
1349 return 0;
1350 }
1351
David S. Miller95ec3eb2011-07-06 01:56:38 -07001352 switch (f->type) {
1353 case PACKET_FANOUT_HASH:
1354 default:
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001355 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
Eric Dumazetbc416d92011-10-06 10:28:31 +00001356 skb = ip_check_defrag(skb, IP_DEFRAG_AF_PACKET);
David S. Miller95ec3eb2011-07-06 01:56:38 -07001357 if (!skb)
1358 return 0;
1359 }
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001360 idx = fanout_demux_hash(f, skb, num);
David S. Miller95ec3eb2011-07-06 01:56:38 -07001361 break;
1362 case PACKET_FANOUT_LB:
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001363 idx = fanout_demux_lb(f, skb, num);
David S. Miller95ec3eb2011-07-06 01:56:38 -07001364 break;
1365 case PACKET_FANOUT_CPU:
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001366 idx = fanout_demux_cpu(f, skb, num);
1367 break;
Daniel Borkmann5df0ddf2013-08-28 22:13:09 +02001368 case PACKET_FANOUT_RND:
1369 idx = fanout_demux_rnd(f, skb, num);
1370 break;
Neil Horman2d360972014-01-22 16:01:44 -05001371 case PACKET_FANOUT_QM:
1372 idx = fanout_demux_qm(f, skb, num);
1373 break;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001374 case PACKET_FANOUT_ROLLOVER:
1375 idx = fanout_demux_rollover(f, skb, 0, (unsigned int) -1, num);
David S. Miller95ec3eb2011-07-06 01:56:38 -07001376 break;
David S. Miller7736d332011-07-05 01:43:20 -07001377 }
1378
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001379 po = pkt_sk(f->arr[idx]);
1380 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER) &&
1381 unlikely(!packet_rcv_has_room(po, skb))) {
1382 idx = fanout_demux_rollover(f, skb, idx, idx, num);
1383 po = pkt_sk(f->arr[idx]);
1384 }
David S. Millerdc99f602011-07-05 01:45:05 -07001385
1386 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1387}
1388
Pavel Emelyanovfff33212012-08-16 05:36:48 +00001389DEFINE_MUTEX(fanout_mutex);
1390EXPORT_SYMBOL_GPL(fanout_mutex);
David S. Millerdc99f602011-07-05 01:45:05 -07001391static LIST_HEAD(fanout_list);
1392
1393static void __fanout_link(struct sock *sk, struct packet_sock *po)
1394{
1395 struct packet_fanout *f = po->fanout;
1396
1397 spin_lock(&f->lock);
1398 f->arr[f->num_members] = sk;
1399 smp_wmb();
1400 f->num_members++;
1401 spin_unlock(&f->lock);
1402}
1403
1404static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1405{
1406 struct packet_fanout *f = po->fanout;
1407 int i;
1408
1409 spin_lock(&f->lock);
1410 for (i = 0; i < f->num_members; i++) {
1411 if (f->arr[i] == sk)
1412 break;
1413 }
1414 BUG_ON(i >= f->num_members);
1415 f->arr[i] = f->arr[f->num_members - 1];
1416 f->num_members--;
1417 spin_unlock(&f->lock);
1418}
1419
Weilong Chend4dd8ae2013-12-23 11:31:38 +08001420static bool match_fanout_group(struct packet_type *ptype, struct sock *sk)
Eric Leblondc0de08d2012-08-16 22:02:58 +00001421{
Weilong Chend4dd8ae2013-12-23 11:31:38 +08001422 if (ptype->af_packet_priv == (void *)((struct packet_sock *)sk)->fanout)
Eric Leblondc0de08d2012-08-16 22:02:58 +00001423 return true;
1424
1425 return false;
1426}
1427
David S. Miller7736d332011-07-05 01:43:20 -07001428static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
David S. Millerdc99f602011-07-05 01:45:05 -07001429{
1430 struct packet_sock *po = pkt_sk(sk);
1431 struct packet_fanout *f, *match;
David S. Miller7736d332011-07-05 01:43:20 -07001432 u8 type = type_flags & 0xff;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001433 u8 flags = type_flags >> 8;
David S. Millerdc99f602011-07-05 01:45:05 -07001434 int err;
1435
1436 switch (type) {
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001437 case PACKET_FANOUT_ROLLOVER:
1438 if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1439 return -EINVAL;
David S. Millerdc99f602011-07-05 01:45:05 -07001440 case PACKET_FANOUT_HASH:
1441 case PACKET_FANOUT_LB:
David S. Miller95ec3eb2011-07-06 01:56:38 -07001442 case PACKET_FANOUT_CPU:
Daniel Borkmann5df0ddf2013-08-28 22:13:09 +02001443 case PACKET_FANOUT_RND:
Neil Horman2d360972014-01-22 16:01:44 -05001444 case PACKET_FANOUT_QM:
David S. Millerdc99f602011-07-05 01:45:05 -07001445 break;
1446 default:
1447 return -EINVAL;
1448 }
1449
1450 if (!po->running)
1451 return -EINVAL;
1452
1453 if (po->fanout)
1454 return -EALREADY;
1455
1456 mutex_lock(&fanout_mutex);
1457 match = NULL;
1458 list_for_each_entry(f, &fanout_list, list) {
1459 if (f->id == id &&
1460 read_pnet(&f->net) == sock_net(sk)) {
1461 match = f;
1462 break;
1463 }
1464 }
Eric Dumazetafe62c62011-07-07 06:41:29 -07001465 err = -EINVAL;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001466 if (match && match->flags != flags)
Eric Dumazetafe62c62011-07-07 06:41:29 -07001467 goto out;
David S. Millerdc99f602011-07-05 01:45:05 -07001468 if (!match) {
Eric Dumazetafe62c62011-07-07 06:41:29 -07001469 err = -ENOMEM;
David S. Millerdc99f602011-07-05 01:45:05 -07001470 match = kzalloc(sizeof(*match), GFP_KERNEL);
Eric Dumazetafe62c62011-07-07 06:41:29 -07001471 if (!match)
1472 goto out;
1473 write_pnet(&match->net, sock_net(sk));
1474 match->id = id;
1475 match->type = type;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001476 match->flags = flags;
Eric Dumazetafe62c62011-07-07 06:41:29 -07001477 atomic_set(&match->rr_cur, 0);
1478 INIT_LIST_HEAD(&match->list);
1479 spin_lock_init(&match->lock);
1480 atomic_set(&match->sk_ref, 0);
1481 match->prot_hook.type = po->prot_hook.type;
1482 match->prot_hook.dev = po->prot_hook.dev;
1483 match->prot_hook.func = packet_rcv_fanout;
1484 match->prot_hook.af_packet_priv = match;
Eric Leblondc0de08d2012-08-16 22:02:58 +00001485 match->prot_hook.id_match = match_fanout_group;
Eric Dumazetafe62c62011-07-07 06:41:29 -07001486 dev_add_pack(&match->prot_hook);
1487 list_add(&match->list, &fanout_list);
1488 }
1489 err = -EINVAL;
1490 if (match->type == type &&
1491 match->prot_hook.type == po->prot_hook.type &&
1492 match->prot_hook.dev == po->prot_hook.dev) {
1493 err = -ENOSPC;
1494 if (atomic_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
1495 __dev_remove_pack(&po->prot_hook);
1496 po->fanout = match;
1497 atomic_inc(&match->sk_ref);
1498 __fanout_link(sk, po);
1499 err = 0;
David S. Millerdc99f602011-07-05 01:45:05 -07001500 }
1501 }
Eric Dumazetafe62c62011-07-07 06:41:29 -07001502out:
David S. Millerdc99f602011-07-05 01:45:05 -07001503 mutex_unlock(&fanout_mutex);
1504 return err;
1505}
1506
1507static void fanout_release(struct sock *sk)
1508{
1509 struct packet_sock *po = pkt_sk(sk);
1510 struct packet_fanout *f;
1511
1512 f = po->fanout;
1513 if (!f)
1514 return;
1515
Pavel Emelyanovfff33212012-08-16 05:36:48 +00001516 mutex_lock(&fanout_mutex);
David S. Millerdc99f602011-07-05 01:45:05 -07001517 po->fanout = NULL;
1518
David S. Millerdc99f602011-07-05 01:45:05 -07001519 if (atomic_dec_and_test(&f->sk_ref)) {
1520 list_del(&f->list);
1521 dev_remove_pack(&f->prot_hook);
1522 kfree(f);
1523 }
1524 mutex_unlock(&fanout_mutex);
1525}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001526
Eric Dumazet90ddc4f2005-12-22 12:49:22 -08001527static const struct proto_ops packet_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001528
Eric Dumazet90ddc4f2005-12-22 12:49:22 -08001529static const struct proto_ops packet_ops_spkt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001530
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001531static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1532 struct packet_type *pt, struct net_device *orig_dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001533{
1534 struct sock *sk;
1535 struct sockaddr_pkt *spkt;
1536
1537 /*
1538 * When we registered the protocol we saved the socket in the data
1539 * field for just this event.
1540 */
1541
1542 sk = pt->af_packet_priv;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001543
Linus Torvalds1da177e2005-04-16 15:20:36 -07001544 /*
1545 * Yank back the headers [hope the device set this
1546 * right or kerboom...]
1547 *
1548 * Incoming packets have ll header pulled,
1549 * push it back.
1550 *
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -07001551 * For outgoing ones skb->data == skb_mac_header(skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001552 * so that this procedure is noop.
1553 */
1554
1555 if (skb->pkt_type == PACKET_LOOPBACK)
1556 goto out;
1557
Octavian Purdila09ad9bc2009-11-25 15:14:13 -08001558 if (!net_eq(dev_net(dev), sock_net(sk)))
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08001559 goto out;
1560
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001561 skb = skb_share_check(skb, GFP_ATOMIC);
1562 if (skb == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001563 goto oom;
1564
1565 /* drop any routing info */
Eric Dumazetadf30902009-06-02 05:19:30 +00001566 skb_dst_drop(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001567
Phil Oester84531c22005-07-12 11:57:52 -07001568 /* drop conntrack reference */
1569 nf_reset(skb);
1570
Herbert Xuffbc6112007-02-04 23:33:10 -08001571 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001572
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -07001573 skb_push(skb, skb->data - skb_mac_header(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001574
1575 /*
1576 * The SOCK_PACKET socket receives _all_ frames.
1577 */
1578
1579 spkt->spkt_family = dev->type;
1580 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1581 spkt->spkt_protocol = skb->protocol;
1582
1583 /*
1584 * Charge the memory to the socket. This is done specifically
1585 * to prevent sockets using all the memory up.
1586 */
1587
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001588 if (sock_queue_rcv_skb(sk, skb) == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001589 return 0;
1590
1591out:
1592 kfree_skb(skb);
1593oom:
1594 return 0;
1595}
1596
1597
1598/*
1599 * Output a raw packet to a device layer. This bypasses all the other
1600 * protocol layers and you must therefore supply it with a complete frame
1601 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001602
Linus Torvalds1da177e2005-04-16 15:20:36 -07001603static int packet_sendmsg_spkt(struct kiocb *iocb, struct socket *sock,
1604 struct msghdr *msg, size_t len)
1605{
1606 struct sock *sk = sock->sk;
Steffen Hurrle342dfc32014-01-17 22:53:15 +01001607 DECLARE_SOCKADDR(struct sockaddr_pkt *, saddr, msg->msg_name);
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001608 struct sk_buff *skb = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001609 struct net_device *dev;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001610 __be16 proto = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001611 int err;
Ben Greear3bdc0eb2012-02-11 15:39:30 +00001612 int extra_len = 0;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001613
Linus Torvalds1da177e2005-04-16 15:20:36 -07001614 /*
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001615 * Get and verify the address.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001616 */
1617
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001618 if (saddr) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001619 if (msg->msg_namelen < sizeof(struct sockaddr))
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001620 return -EINVAL;
1621 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1622 proto = saddr->spkt_protocol;
1623 } else
1624 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001625
1626 /*
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001627 * Find the device first to size check it
Linus Torvalds1da177e2005-04-16 15:20:36 -07001628 */
1629
danborkmann@iogearbox.netde74e922012-06-10 08:59:28 +00001630 saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001631retry:
Eric Dumazet654d1f82009-11-02 10:43:32 +01001632 rcu_read_lock();
1633 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001634 err = -ENODEV;
1635 if (dev == NULL)
1636 goto out_unlock;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001637
David S. Millerd5e76b02007-01-25 19:30:36 -08001638 err = -ENETDOWN;
1639 if (!(dev->flags & IFF_UP))
1640 goto out_unlock;
1641
Linus Torvalds1da177e2005-04-16 15:20:36 -07001642 /*
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001643 * You may not queue a frame bigger than the mtu. This is the lowest level
1644 * raw protocol and you must do your own fragmentation at this level.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001645 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001646
Ben Greear3bdc0eb2012-02-11 15:39:30 +00001647 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1648 if (!netif_supports_nofcs(dev)) {
1649 err = -EPROTONOSUPPORT;
1650 goto out_unlock;
1651 }
1652 extra_len = 4; /* We're doing our own CRC */
1653 }
1654
Linus Torvalds1da177e2005-04-16 15:20:36 -07001655 err = -EMSGSIZE;
Ben Greear3bdc0eb2012-02-11 15:39:30 +00001656 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001657 goto out_unlock;
1658
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001659 if (!skb) {
1660 size_t reserved = LL_RESERVED_SPACE(dev);
Herbert Xu4ce40912011-11-18 02:20:05 +00001661 int tlen = dev->needed_tailroom;
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001662 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001663
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001664 rcu_read_unlock();
Herbert Xu4ce40912011-11-18 02:20:05 +00001665 skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001666 if (skb == NULL)
1667 return -ENOBUFS;
1668 /* FIXME: Save some space for broken drivers that write a hard
1669 * header at transmission time by themselves. PPP is the notable
1670 * one here. This should really be fixed at the driver level.
1671 */
1672 skb_reserve(skb, reserved);
1673 skb_reset_network_header(skb);
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001674
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001675 /* Try to align data part correctly */
1676 if (hhlen) {
1677 skb->data -= hhlen;
1678 skb->tail -= hhlen;
1679 if (len < hhlen)
1680 skb_reset_network_header(skb);
1681 }
1682 err = memcpy_fromiovec(skb_put(skb, len), msg->msg_iov, len);
1683 if (err)
1684 goto out_free;
1685 goto retry;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001686 }
1687
Ben Greear3bdc0eb2012-02-11 15:39:30 +00001688 if (len > (dev->mtu + dev->hard_header_len + extra_len)) {
Ben Greear57f89bf2011-02-11 09:35:18 +00001689 /* Earlier code assumed this would be a VLAN pkt,
1690 * double-check this now that we have the actual
1691 * packet in hand.
1692 */
1693 struct ethhdr *ehdr;
1694 skb_reset_mac_header(skb);
1695 ehdr = eth_hdr(skb);
1696 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
1697 err = -EMSGSIZE;
1698 goto out_unlock;
1699 }
1700 }
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001701
Linus Torvalds1da177e2005-04-16 15:20:36 -07001702 skb->protocol = proto;
1703 skb->dev = dev;
1704 skb->priority = sk->sk_priority;
Eric Dumazet2d37a182009-10-01 19:14:46 +00001705 skb->mark = sk->sk_mark;
Daniel Borkmannbf84a012013-04-14 08:08:13 +00001706
1707 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001708
Ben Greear3bdc0eb2012-02-11 15:39:30 +00001709 if (unlikely(extra_len == 4))
1710 skb->no_fcs = 1;
1711
Jason Wang40893fd2013-03-26 23:11:22 +00001712 skb_probe_transport_header(skb, 0);
Jason Wangc1aad272013-03-25 20:19:57 +00001713
Linus Torvalds1da177e2005-04-16 15:20:36 -07001714 dev_queue_xmit(skb);
Eric Dumazet654d1f82009-11-02 10:43:32 +01001715 rcu_read_unlock();
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001716 return len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001717
Linus Torvalds1da177e2005-04-16 15:20:36 -07001718out_unlock:
Eric Dumazet654d1f82009-11-02 10:43:32 +01001719 rcu_read_unlock();
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001720out_free:
1721 kfree_skb(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001722 return err;
1723}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001724
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001725static unsigned int run_filter(const struct sk_buff *skb,
Eric Dumazet62ab0812010-12-06 20:50:09 +00001726 const struct sock *sk,
David S. Millerdbcb5852007-01-24 15:21:02 -08001727 unsigned int res)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001728{
1729 struct sk_filter *filter;
1730
Eric Dumazet80f8f102011-01-18 07:46:52 +00001731 rcu_read_lock();
1732 filter = rcu_dereference(sk->sk_filter);
David S. Millerdbcb5852007-01-24 15:21:02 -08001733 if (filter != NULL)
Eric Dumazet0a148422011-04-20 09:27:32 +00001734 res = SK_RUN_FILTER(filter, skb);
Eric Dumazet80f8f102011-01-18 07:46:52 +00001735 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001736
David S. Millerdbcb5852007-01-24 15:21:02 -08001737 return res;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001738}
1739
1740/*
Eric Dumazet62ab0812010-12-06 20:50:09 +00001741 * This function makes lazy skb cloning in hope that most of packets
1742 * are discarded by BPF.
1743 *
1744 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
1745 * and skb->cb are mangled. It works because (and until) packets
1746 * falling here are owned by current CPU. Output packets are cloned
1747 * by dev_queue_xmit_nit(), input packets are processed by net_bh
1748 * sequencially, so that if we return skb to original state on exit,
1749 * we will not harm anyone.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001750 */
1751
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001752static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
1753 struct packet_type *pt, struct net_device *orig_dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001754{
1755 struct sock *sk;
1756 struct sockaddr_ll *sll;
1757 struct packet_sock *po;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001758 u8 *skb_head = skb->data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001759 int skb_len = skb->len;
David S. Millerdbcb5852007-01-24 15:21:02 -08001760 unsigned int snaplen, res;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001761
1762 if (skb->pkt_type == PACKET_LOOPBACK)
1763 goto drop;
1764
1765 sk = pt->af_packet_priv;
1766 po = pkt_sk(sk);
1767
Octavian Purdila09ad9bc2009-11-25 15:14:13 -08001768 if (!net_eq(dev_net(dev), sock_net(sk)))
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08001769 goto drop;
1770
Linus Torvalds1da177e2005-04-16 15:20:36 -07001771 skb->dev = dev;
1772
Stephen Hemminger3b04ddd2007-10-09 01:40:57 -07001773 if (dev->header_ops) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001774 /* The device has an explicit notion of ll header,
Eric Dumazet62ab0812010-12-06 20:50:09 +00001775 * exported to higher levels.
1776 *
1777 * Otherwise, the device hides details of its frame
1778 * structure, so that corresponding packet head is
1779 * never delivered to user.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001780 */
1781 if (sk->sk_type != SOCK_DGRAM)
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -07001782 skb_push(skb, skb->data - skb_mac_header(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001783 else if (skb->pkt_type == PACKET_OUTGOING) {
1784 /* Special case: outgoing packets have ll header at head */
Arnaldo Carvalho de Melobbe735e2007-03-10 22:16:10 -03001785 skb_pull(skb, skb_network_offset(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001786 }
1787 }
1788
1789 snaplen = skb->len;
1790
David S. Millerdbcb5852007-01-24 15:21:02 -08001791 res = run_filter(skb, sk, snaplen);
1792 if (!res)
Dmitry Mishinfda9ef52006-08-31 15:28:39 -07001793 goto drop_n_restore;
David S. Millerdbcb5852007-01-24 15:21:02 -08001794 if (snaplen > res)
1795 snaplen = res;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001796
Eric Dumazet0fd7bac2011-12-21 07:11:44 +00001797 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001798 goto drop_n_acct;
1799
1800 if (skb_shared(skb)) {
1801 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
1802 if (nskb == NULL)
1803 goto drop_n_acct;
1804
1805 if (skb_head != skb->data) {
1806 skb->data = skb_head;
1807 skb->len = skb_len;
1808 }
Eric Dumazetabc4e4f2012-04-19 02:24:42 +00001809 consume_skb(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001810 skb = nskb;
1811 }
1812
Herbert Xuffbc6112007-02-04 23:33:10 -08001813 BUILD_BUG_ON(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8 >
1814 sizeof(skb->cb));
1815
1816 sll = &PACKET_SKB_CB(skb)->sa.ll;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001817 sll->sll_family = AF_PACKET;
1818 sll->sll_hatype = dev->type;
1819 sll->sll_protocol = skb->protocol;
1820 sll->sll_pkttype = skb->pkt_type;
Peter P Waskiewicz Jr8032b462007-11-10 22:03:25 -08001821 if (unlikely(po->origdev))
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07001822 sll->sll_ifindex = orig_dev->ifindex;
1823 else
1824 sll->sll_ifindex = dev->ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001825
Stephen Hemmingerb95cce32007-09-26 22:13:38 -07001826 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001827
Herbert Xuffbc6112007-02-04 23:33:10 -08001828 PACKET_SKB_CB(skb)->origlen = skb->len;
Herbert Xu8dc41942007-02-04 23:31:32 -08001829
Linus Torvalds1da177e2005-04-16 15:20:36 -07001830 if (pskb_trim(skb, snaplen))
1831 goto drop_n_acct;
1832
1833 skb_set_owner_r(skb, sk);
1834 skb->dev = NULL;
Eric Dumazetadf30902009-06-02 05:19:30 +00001835 skb_dst_drop(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001836
Phil Oester84531c22005-07-12 11:57:52 -07001837 /* drop conntrack reference */
1838 nf_reset(skb);
1839
Linus Torvalds1da177e2005-04-16 15:20:36 -07001840 spin_lock(&sk->sk_receive_queue.lock);
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00001841 po->stats.stats1.tp_packets++;
Neil Horman3b885782009-10-12 13:26:31 -07001842 skb->dropcount = atomic_read(&sk->sk_drops);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001843 __skb_queue_tail(&sk->sk_receive_queue, skb);
1844 spin_unlock(&sk->sk_receive_queue.lock);
David S. Miller676d2362014-04-11 16:15:36 -04001845 sk->sk_data_ready(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001846 return 0;
1847
1848drop_n_acct:
Willem de Bruijn7091fbd2011-09-30 10:38:28 +00001849 spin_lock(&sk->sk_receive_queue.lock);
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00001850 po->stats.stats1.tp_drops++;
Willem de Bruijn7091fbd2011-09-30 10:38:28 +00001851 atomic_inc(&sk->sk_drops);
1852 spin_unlock(&sk->sk_receive_queue.lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001853
1854drop_n_restore:
1855 if (skb_head != skb->data && skb_shared(skb)) {
1856 skb->data = skb_head;
1857 skb->len = skb_len;
1858 }
1859drop:
Neil Hormanead2ceb2009-03-11 09:49:55 +00001860 consume_skb(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001861 return 0;
1862}
1863
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001864static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
1865 struct packet_type *pt, struct net_device *orig_dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001866{
1867 struct sock *sk;
1868 struct packet_sock *po;
1869 struct sockaddr_ll *sll;
Daniel Borkmann184f4892013-04-16 01:57:46 +00001870 union tpacket_uhdr h;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001871 u8 *skb_head = skb->data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001872 int skb_len = skb->len;
David S. Millerdbcb5852007-01-24 15:21:02 -08001873 unsigned int snaplen, res;
chetan lokef6fb8f102011-08-19 10:18:16 +00001874 unsigned long status = TP_STATUS_USER;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07001875 unsigned short macoff, netoff, hdrlen;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001876 struct sk_buff *copy_skb = NULL;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07001877 struct timespec ts;
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +00001878 __u32 ts_status;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001879
Atzm Watanabe51846352013-12-17 22:53:32 +09001880 /* struct tpacket{2,3}_hdr is aligned to a multiple of TPACKET_ALIGNMENT.
1881 * We may add members to them until current aligned size without forcing
1882 * userspace to call getsockopt(..., PACKET_HDRLEN, ...).
1883 */
1884 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h2)) != 32);
1885 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h3)) != 48);
1886
Linus Torvalds1da177e2005-04-16 15:20:36 -07001887 if (skb->pkt_type == PACKET_LOOPBACK)
1888 goto drop;
1889
1890 sk = pt->af_packet_priv;
1891 po = pkt_sk(sk);
1892
Octavian Purdila09ad9bc2009-11-25 15:14:13 -08001893 if (!net_eq(dev_net(dev), sock_net(sk)))
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08001894 goto drop;
1895
Stephen Hemminger3b04ddd2007-10-09 01:40:57 -07001896 if (dev->header_ops) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001897 if (sk->sk_type != SOCK_DGRAM)
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -07001898 skb_push(skb, skb->data - skb_mac_header(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001899 else if (skb->pkt_type == PACKET_OUTGOING) {
1900 /* Special case: outgoing packets have ll header at head */
Arnaldo Carvalho de Melobbe735e2007-03-10 22:16:10 -03001901 skb_pull(skb, skb_network_offset(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001902 }
1903 }
1904
Herbert Xu8dc41942007-02-04 23:31:32 -08001905 if (skb->ip_summed == CHECKSUM_PARTIAL)
1906 status |= TP_STATUS_CSUMNOTREADY;
1907
Linus Torvalds1da177e2005-04-16 15:20:36 -07001908 snaplen = skb->len;
1909
David S. Millerdbcb5852007-01-24 15:21:02 -08001910 res = run_filter(skb, sk, snaplen);
1911 if (!res)
Dmitry Mishinfda9ef52006-08-31 15:28:39 -07001912 goto drop_n_restore;
David S. Millerdbcb5852007-01-24 15:21:02 -08001913 if (snaplen > res)
1914 snaplen = res;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001915
1916 if (sk->sk_type == SOCK_DGRAM) {
Patrick McHardy8913336a2008-07-18 18:05:19 -07001917 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
1918 po->tp_reserve;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001919 } else {
Eric Dumazet95c96172012-04-15 05:58:06 +00001920 unsigned int maclen = skb_network_offset(skb);
Patrick McHardybbd6ef82008-07-14 22:50:15 -07001921 netoff = TPACKET_ALIGN(po->tp_hdrlen +
Patrick McHardy8913336a2008-07-18 18:05:19 -07001922 (maclen < 16 ? 16 : maclen)) +
1923 po->tp_reserve;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001924 macoff = netoff - maclen;
1925 }
chetan lokef6fb8f102011-08-19 10:18:16 +00001926 if (po->tp_version <= TPACKET_V2) {
1927 if (macoff + snaplen > po->rx_ring.frame_size) {
1928 if (po->copy_thresh &&
Eric Dumazet0fd7bac2011-12-21 07:11:44 +00001929 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
chetan lokef6fb8f102011-08-19 10:18:16 +00001930 if (skb_shared(skb)) {
1931 copy_skb = skb_clone(skb, GFP_ATOMIC);
1932 } else {
1933 copy_skb = skb_get(skb);
1934 skb_head = skb->data;
1935 }
1936 if (copy_skb)
1937 skb_set_owner_r(copy_skb, sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001938 }
chetan lokef6fb8f102011-08-19 10:18:16 +00001939 snaplen = po->rx_ring.frame_size - macoff;
1940 if ((int)snaplen < 0)
1941 snaplen = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001942 }
Eric Dumazetdc808112014-08-15 09:16:04 -07001943 } else if (unlikely(macoff + snaplen >
1944 GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len)) {
1945 u32 nval;
1946
1947 nval = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len - macoff;
1948 pr_err_once("tpacket_rcv: packet too big, clamped from %u to %u. macoff=%u\n",
1949 snaplen, nval, macoff);
1950 snaplen = nval;
1951 if (unlikely((int)snaplen < 0)) {
1952 snaplen = 0;
1953 macoff = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len;
1954 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001955 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001956 spin_lock(&sk->sk_receive_queue.lock);
chetan lokef6fb8f102011-08-19 10:18:16 +00001957 h.raw = packet_current_rx_frame(po, skb,
1958 TP_STATUS_KERNEL, (macoff+snaplen));
Patrick McHardybbd6ef82008-07-14 22:50:15 -07001959 if (!h.raw)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001960 goto ring_is_full;
chetan lokef6fb8f102011-08-19 10:18:16 +00001961 if (po->tp_version <= TPACKET_V2) {
1962 packet_increment_rx_head(po, &po->rx_ring);
1963 /*
1964 * LOSING will be reported till you read the stats,
1965 * because it's COR - Clear On Read.
1966 * Anyways, moving it for V1/V2 only as V3 doesn't need this
1967 * at packet level.
1968 */
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00001969 if (po->stats.stats1.tp_drops)
chetan lokef6fb8f102011-08-19 10:18:16 +00001970 status |= TP_STATUS_LOSING;
1971 }
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00001972 po->stats.stats1.tp_packets++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001973 if (copy_skb) {
1974 status |= TP_STATUS_COPY;
1975 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
1976 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001977 spin_unlock(&sk->sk_receive_queue.lock);
1978
Patrick McHardybbd6ef82008-07-14 22:50:15 -07001979 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +00001980
1981 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
Daniel Borkmann7a513842013-04-23 00:39:29 +00001982 getnstimeofday(&ts);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001983
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +00001984 status |= ts_status;
1985
Patrick McHardybbd6ef82008-07-14 22:50:15 -07001986 switch (po->tp_version) {
1987 case TPACKET_V1:
1988 h.h1->tp_len = skb->len;
1989 h.h1->tp_snaplen = snaplen;
1990 h.h1->tp_mac = macoff;
1991 h.h1->tp_net = netoff;
Daniel Borkmann4b457bd2013-04-16 01:29:11 +00001992 h.h1->tp_sec = ts.tv_sec;
1993 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07001994 hdrlen = sizeof(*h.h1);
1995 break;
1996 case TPACKET_V2:
1997 h.h2->tp_len = skb->len;
1998 h.h2->tp_snaplen = snaplen;
1999 h.h2->tp_mac = macoff;
2000 h.h2->tp_net = netoff;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07002001 h.h2->tp_sec = ts.tv_sec;
2002 h.h2->tp_nsec = ts.tv_nsec;
Ben Greeara3bcc232011-06-01 06:49:10 +00002003 if (vlan_tx_tag_present(skb)) {
2004 h.h2->tp_vlan_tci = vlan_tx_tag_get(skb);
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +09002005 h.h2->tp_vlan_tpid = ntohs(skb->vlan_proto);
2006 status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
Ben Greeara3bcc232011-06-01 06:49:10 +00002007 } else {
2008 h.h2->tp_vlan_tci = 0;
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +09002009 h.h2->tp_vlan_tpid = 0;
Ben Greeara3bcc232011-06-01 06:49:10 +00002010 }
Atzm Watanabee4d26f42013-12-17 22:53:36 +09002011 memset(h.h2->tp_padding, 0, sizeof(h.h2->tp_padding));
Patrick McHardybbd6ef82008-07-14 22:50:15 -07002012 hdrlen = sizeof(*h.h2);
2013 break;
chetan lokef6fb8f102011-08-19 10:18:16 +00002014 case TPACKET_V3:
2015 /* tp_nxt_offset,vlan are already populated above.
2016 * So DONT clear those fields here
2017 */
2018 h.h3->tp_status |= status;
2019 h.h3->tp_len = skb->len;
2020 h.h3->tp_snaplen = snaplen;
2021 h.h3->tp_mac = macoff;
2022 h.h3->tp_net = netoff;
chetan lokef6fb8f102011-08-19 10:18:16 +00002023 h.h3->tp_sec = ts.tv_sec;
2024 h.h3->tp_nsec = ts.tv_nsec;
Atzm Watanabee4d26f42013-12-17 22:53:36 +09002025 memset(h.h3->tp_padding, 0, sizeof(h.h3->tp_padding));
chetan lokef6fb8f102011-08-19 10:18:16 +00002026 hdrlen = sizeof(*h.h3);
2027 break;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07002028 default:
2029 BUG();
2030 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002031
Patrick McHardybbd6ef82008-07-14 22:50:15 -07002032 sll = h.raw + TPACKET_ALIGN(hdrlen);
Stephen Hemmingerb95cce32007-09-26 22:13:38 -07002033 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002034 sll->sll_family = AF_PACKET;
2035 sll->sll_hatype = dev->type;
2036 sll->sll_protocol = skb->protocol;
2037 sll->sll_pkttype = skb->pkt_type;
Peter P Waskiewicz Jr8032b462007-11-10 22:03:25 -08002038 if (unlikely(po->origdev))
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07002039 sll->sll_ifindex = orig_dev->ifindex;
2040 else
2041 sll->sll_ifindex = dev->ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002042
Ralf Baechlee16aa202006-12-07 00:11:33 -08002043 smp_mb();
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01002044
Changli Gaof6dafa92010-12-07 04:26:16 +00002045#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01002046 if (po->tp_version <= TPACKET_V2) {
Changli Gao0af55bb2010-12-01 02:52:20 +00002047 u8 *start, *end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002048
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01002049 end = (u8 *) PAGE_ALIGN((unsigned long) h.raw +
2050 macoff + snaplen);
2051
2052 for (start = h.raw; start < end; start += PAGE_SIZE)
2053 flush_dcache_page(pgv_to_page(start));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002054 }
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01002055 smp_wmb();
Changli Gaof6dafa92010-12-07 04:26:16 +00002056#endif
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01002057
chetan lokef6fb8f102011-08-19 10:18:16 +00002058 if (po->tp_version <= TPACKET_V2)
2059 __packet_set_status(po, h.raw, status);
2060 else
2061 prb_clear_blk_fill_status(&po->rx_ring);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002062
David S. Miller676d2362014-04-11 16:15:36 -04002063 sk->sk_data_ready(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002064
2065drop_n_restore:
2066 if (skb_head != skb->data && skb_shared(skb)) {
2067 skb->data = skb_head;
2068 skb->len = skb_len;
2069 }
2070drop:
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002071 kfree_skb(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002072 return 0;
2073
2074ring_is_full:
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00002075 po->stats.stats1.tp_drops++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002076 spin_unlock(&sk->sk_receive_queue.lock);
2077
David S. Miller676d2362014-04-11 16:15:36 -04002078 sk->sk_data_ready(sk);
Wei Yongjunacb5d752009-02-25 00:36:42 +00002079 kfree_skb(copy_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002080 goto drop_n_restore;
2081}
2082
Johann Baudy69e3c752009-05-18 22:11:22 -07002083static void tpacket_destruct_skb(struct sk_buff *skb)
2084{
2085 struct packet_sock *po = pkt_sk(skb->sk);
Johann Baudy69e3c752009-05-18 22:11:22 -07002086
Johann Baudy69e3c752009-05-18 22:11:22 -07002087 if (likely(po->tx_ring.pg_vec)) {
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01002088 void *ph;
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +00002089 __u32 ts;
2090
Johann Baudy69e3c752009-05-18 22:11:22 -07002091 ph = skb_shinfo(skb)->destructor_arg;
Daniel Borkmannb0138402014-01-15 16:25:36 +01002092 packet_dec_pending(&po->tx_ring);
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +00002093
2094 ts = __packet_set_timestamp(po, ph, skb);
2095 __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
Johann Baudy69e3c752009-05-18 22:11:22 -07002096 }
2097
2098 sock_wfree(skb);
2099}
2100
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002101static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
2102 void *frame, struct net_device *dev, int size_max,
Herbert Xuae641942011-11-18 02:20:04 +00002103 __be16 proto, unsigned char *addr, int hlen)
Johann Baudy69e3c752009-05-18 22:11:22 -07002104{
Daniel Borkmann184f4892013-04-16 01:57:46 +00002105 union tpacket_uhdr ph;
David S. Miller09effa62013-08-07 17:11:00 -07002106 int to_write, offset, len, tp_len, nr_frags, len_max;
Johann Baudy69e3c752009-05-18 22:11:22 -07002107 struct socket *sock = po->sk.sk_socket;
2108 struct page *page;
2109 void *data;
2110 int err;
2111
2112 ph.raw = frame;
2113
2114 skb->protocol = proto;
2115 skb->dev = dev;
2116 skb->priority = po->sk.sk_priority;
Eric Dumazet2d37a182009-10-01 19:14:46 +00002117 skb->mark = po->sk.sk_mark;
Willem de Bruijn2e313962013-04-23 00:39:28 +00002118 sock_tx_timestamp(&po->sk, &skb_shinfo(skb)->tx_flags);
Johann Baudy69e3c752009-05-18 22:11:22 -07002119 skb_shinfo(skb)->destructor_arg = ph.raw;
2120
2121 switch (po->tp_version) {
2122 case TPACKET_V2:
2123 tp_len = ph.h2->tp_len;
2124 break;
2125 default:
2126 tp_len = ph.h1->tp_len;
2127 break;
2128 }
David S. Miller09effa62013-08-07 17:11:00 -07002129 if (unlikely(tp_len > size_max)) {
2130 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
2131 return -EMSGSIZE;
2132 }
Johann Baudy69e3c752009-05-18 22:11:22 -07002133
Herbert Xuae641942011-11-18 02:20:04 +00002134 skb_reserve(skb, hlen);
Johann Baudy69e3c752009-05-18 22:11:22 -07002135 skb_reset_network_header(skb);
Jason Wangc1aad272013-03-25 20:19:57 +00002136
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01002137 if (!packet_use_direct_xmit(po))
2138 skb_probe_transport_header(skb, 0);
2139 if (unlikely(po->tp_tx_has_off)) {
Paul Chavent5920cd3a2012-11-06 23:10:47 +00002140 int off_min, off_max, off;
2141 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2142 off_max = po->tx_ring.frame_size - tp_len;
2143 if (sock->type == SOCK_DGRAM) {
2144 switch (po->tp_version) {
2145 case TPACKET_V2:
2146 off = ph.h2->tp_net;
2147 break;
2148 default:
2149 off = ph.h1->tp_net;
2150 break;
2151 }
2152 } else {
2153 switch (po->tp_version) {
2154 case TPACKET_V2:
2155 off = ph.h2->tp_mac;
2156 break;
2157 default:
2158 off = ph.h1->tp_mac;
2159 break;
2160 }
2161 }
2162 if (unlikely((off < off_min) || (off_max < off)))
2163 return -EINVAL;
2164 data = ph.raw + off;
2165 } else {
2166 data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll);
2167 }
Johann Baudy69e3c752009-05-18 22:11:22 -07002168 to_write = tp_len;
2169
2170 if (sock->type == SOCK_DGRAM) {
2171 err = dev_hard_header(skb, dev, ntohs(proto), addr,
2172 NULL, tp_len);
2173 if (unlikely(err < 0))
2174 return -EINVAL;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002175 } else if (dev->hard_header_len) {
Johann Baudy69e3c752009-05-18 22:11:22 -07002176 /* net device doesn't like empty head */
2177 if (unlikely(tp_len <= dev->hard_header_len)) {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002178 pr_err("packet size is too short (%d < %d)\n",
2179 tp_len, dev->hard_header_len);
Johann Baudy69e3c752009-05-18 22:11:22 -07002180 return -EINVAL;
2181 }
2182
2183 skb_push(skb, dev->hard_header_len);
2184 err = skb_store_bits(skb, 0, data,
2185 dev->hard_header_len);
2186 if (unlikely(err))
2187 return err;
2188
2189 data += dev->hard_header_len;
2190 to_write -= dev->hard_header_len;
2191 }
2192
Johann Baudy69e3c752009-05-18 22:11:22 -07002193 offset = offset_in_page(data);
2194 len_max = PAGE_SIZE - offset;
2195 len = ((to_write > len_max) ? len_max : to_write);
2196
2197 skb->data_len = to_write;
2198 skb->len += to_write;
2199 skb->truesize += to_write;
2200 atomic_add(to_write, &po->sk.sk_wmem_alloc);
2201
2202 while (likely(to_write)) {
2203 nr_frags = skb_shinfo(skb)->nr_frags;
2204
2205 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002206 pr_err("Packet exceed the number of skb frags(%lu)\n",
2207 MAX_SKB_FRAGS);
Johann Baudy69e3c752009-05-18 22:11:22 -07002208 return -EFAULT;
2209 }
2210
Changli Gao0af55bb2010-12-01 02:52:20 +00002211 page = pgv_to_page(data);
2212 data += len;
Johann Baudy69e3c752009-05-18 22:11:22 -07002213 flush_dcache_page(page);
2214 get_page(page);
Changli Gao0af55bb2010-12-01 02:52:20 +00002215 skb_fill_page_desc(skb, nr_frags, page, offset, len);
Johann Baudy69e3c752009-05-18 22:11:22 -07002216 to_write -= len;
2217 offset = 0;
2218 len_max = PAGE_SIZE;
2219 len = ((to_write > len_max) ? len_max : to_write);
2220 }
2221
2222 return tp_len;
2223}
2224
2225static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2226{
Johann Baudy69e3c752009-05-18 22:11:22 -07002227 struct sk_buff *skb;
2228 struct net_device *dev;
2229 __be16 proto;
David S. Miller09effa62013-08-07 17:11:00 -07002230 int err, reserve = 0;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002231 void *ph;
Steffen Hurrle342dfc32014-01-17 22:53:15 +01002232 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
Daniel Borkmann87a2fd22014-01-15 16:25:35 +01002233 bool need_wait = !(msg->msg_flags & MSG_DONTWAIT);
Johann Baudy69e3c752009-05-18 22:11:22 -07002234 int tp_len, size_max;
2235 unsigned char *addr;
2236 int len_sum = 0;
danborkmann@iogearbox.net9e670302012-08-20 03:34:03 +00002237 int status = TP_STATUS_AVAILABLE;
Herbert Xuae641942011-11-18 02:20:04 +00002238 int hlen, tlen;
Johann Baudy69e3c752009-05-18 22:11:22 -07002239
Johann Baudy69e3c752009-05-18 22:11:22 -07002240 mutex_lock(&po->pg_vec_lock);
2241
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01002242 if (likely(saddr == NULL)) {
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002243 dev = packet_cached_dev_get(po);
Johann Baudy69e3c752009-05-18 22:11:22 -07002244 proto = po->num;
2245 addr = NULL;
2246 } else {
2247 err = -EINVAL;
2248 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2249 goto out;
2250 if (msg->msg_namelen < (saddr->sll_halen
2251 + offsetof(struct sockaddr_ll,
2252 sll_addr)))
2253 goto out;
Johann Baudy69e3c752009-05-18 22:11:22 -07002254 proto = saddr->sll_protocol;
2255 addr = saddr->sll_addr;
Ben Greear827d9782011-06-01 07:18:53 +00002256 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
Johann Baudy69e3c752009-05-18 22:11:22 -07002257 }
2258
Johann Baudy69e3c752009-05-18 22:11:22 -07002259 err = -ENXIO;
2260 if (unlikely(dev == NULL))
2261 goto out;
Johann Baudy69e3c752009-05-18 22:11:22 -07002262 err = -ENETDOWN;
2263 if (unlikely(!(dev->flags & IFF_UP)))
2264 goto out_put;
2265
Daniel Borkmann52f14542014-02-28 02:22:06 +01002266 reserve = dev->hard_header_len + VLAN_HLEN;
Johann Baudy69e3c752009-05-18 22:11:22 -07002267 size_max = po->tx_ring.frame_size
Gabor Gombasb5dd8842009-10-29 03:19:11 -07002268 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
Johann Baudy69e3c752009-05-18 22:11:22 -07002269
David S. Miller09effa62013-08-07 17:11:00 -07002270 if (size_max > dev->mtu + reserve)
2271 size_max = dev->mtu + reserve;
2272
Johann Baudy69e3c752009-05-18 22:11:22 -07002273 do {
2274 ph = packet_current_frame(po, &po->tx_ring,
Daniel Borkmann87a2fd22014-01-15 16:25:35 +01002275 TP_STATUS_SEND_REQUEST);
Johann Baudy69e3c752009-05-18 22:11:22 -07002276 if (unlikely(ph == NULL)) {
Daniel Borkmann87a2fd22014-01-15 16:25:35 +01002277 if (need_wait && need_resched())
2278 schedule();
Johann Baudy69e3c752009-05-18 22:11:22 -07002279 continue;
2280 }
2281
2282 status = TP_STATUS_SEND_REQUEST;
Herbert Xuae641942011-11-18 02:20:04 +00002283 hlen = LL_RESERVED_SPACE(dev);
2284 tlen = dev->needed_tailroom;
Johann Baudy69e3c752009-05-18 22:11:22 -07002285 skb = sock_alloc_send_skb(&po->sk,
Herbert Xuae641942011-11-18 02:20:04 +00002286 hlen + tlen + sizeof(struct sockaddr_ll),
Johann Baudy69e3c752009-05-18 22:11:22 -07002287 0, &err);
2288
2289 if (unlikely(skb == NULL))
2290 goto out_status;
2291
2292 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto,
Daniel Borkmann52f14542014-02-28 02:22:06 +01002293 addr, hlen);
2294 if (tp_len > dev->mtu + dev->hard_header_len) {
2295 struct ethhdr *ehdr;
2296 /* Earlier code assumed this would be a VLAN pkt,
2297 * double-check this now that we have the actual
2298 * packet in hand.
2299 */
Johann Baudy69e3c752009-05-18 22:11:22 -07002300
Daniel Borkmann52f14542014-02-28 02:22:06 +01002301 skb_reset_mac_header(skb);
2302 ehdr = eth_hdr(skb);
2303 if (ehdr->h_proto != htons(ETH_P_8021Q))
2304 tp_len = -EMSGSIZE;
2305 }
Johann Baudy69e3c752009-05-18 22:11:22 -07002306 if (unlikely(tp_len < 0)) {
2307 if (po->tp_loss) {
2308 __packet_set_status(po, ph,
2309 TP_STATUS_AVAILABLE);
2310 packet_increment_head(&po->tx_ring);
2311 kfree_skb(skb);
2312 continue;
2313 } else {
2314 status = TP_STATUS_WRONG_FORMAT;
2315 err = tp_len;
2316 goto out_status;
2317 }
2318 }
2319
Daniel Borkmann0fd5d572014-02-16 15:55:22 +01002320 packet_pick_tx_queue(dev, skb);
2321
Johann Baudy69e3c752009-05-18 22:11:22 -07002322 skb->destructor = tpacket_destruct_skb;
2323 __packet_set_status(po, ph, TP_STATUS_SENDING);
Daniel Borkmannb0138402014-01-15 16:25:36 +01002324 packet_inc_pending(&po->tx_ring);
Johann Baudy69e3c752009-05-18 22:11:22 -07002325
2326 status = TP_STATUS_SEND_REQUEST;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01002327 err = po->xmit(skb);
Jarek Poplawskieb70df12010-01-10 22:04:19 +00002328 if (unlikely(err > 0)) {
2329 err = net_xmit_errno(err);
2330 if (err && __packet_get_status(po, ph) ==
2331 TP_STATUS_AVAILABLE) {
2332 /* skb was destructed already */
2333 skb = NULL;
2334 goto out_status;
2335 }
2336 /*
2337 * skb was dropped but not destructed yet;
2338 * let's treat it like congestion or err < 0
2339 */
2340 err = 0;
2341 }
Johann Baudy69e3c752009-05-18 22:11:22 -07002342 packet_increment_head(&po->tx_ring);
2343 len_sum += tp_len;
Daniel Borkmannb0138402014-01-15 16:25:36 +01002344 } while (likely((ph != NULL) ||
2345 /* Note: packet_read_pending() might be slow if we have
2346 * to call it as it's per_cpu variable, but in fast-path
2347 * we already short-circuit the loop with the first
2348 * condition, and luckily don't have to go that path
2349 * anyway.
2350 */
2351 (need_wait && packet_read_pending(&po->tx_ring))));
Johann Baudy69e3c752009-05-18 22:11:22 -07002352
2353 err = len_sum;
2354 goto out_put;
2355
Johann Baudy69e3c752009-05-18 22:11:22 -07002356out_status:
2357 __packet_set_status(po, ph, status);
2358 kfree_skb(skb);
2359out_put:
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002360 dev_put(dev);
Johann Baudy69e3c752009-05-18 22:11:22 -07002361out:
2362 mutex_unlock(&po->pg_vec_lock);
2363 return err;
2364}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002365
Olof Johanssoneea49cc92011-11-02 11:00:49 +00002366static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2367 size_t reserve, size_t len,
2368 size_t linear, int noblock,
2369 int *err)
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002370{
2371 struct sk_buff *skb;
2372
2373 /* Under a page? Don't bother with paged skb. */
2374 if (prepad + len < PAGE_SIZE || !linear)
2375 linear = len;
2376
2377 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
Eric Dumazet28d64272013-08-08 14:38:47 -07002378 err, 0);
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002379 if (!skb)
2380 return NULL;
2381
2382 skb_reserve(skb, reserve);
2383 skb_put(skb, linear);
2384 skb->data_len = len - linear;
2385 skb->len += len - linear;
2386
2387 return skb;
2388}
2389
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01002390static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002391{
2392 struct sock *sk = sock->sk;
Steffen Hurrle342dfc32014-01-17 22:53:15 +01002393 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002394 struct sk_buff *skb;
2395 struct net_device *dev;
Al Viro0e11c912006-11-08 00:26:29 -08002396 __be16 proto;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002397 unsigned char *addr;
Ben Greear827d9782011-06-01 07:18:53 +00002398 int err, reserve = 0;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002399 struct virtio_net_hdr vnet_hdr = { 0 };
2400 int offset = 0;
2401 int vnet_hdr_len;
2402 struct packet_sock *po = pkt_sk(sk);
2403 unsigned short gso_type = 0;
Herbert Xuae641942011-11-18 02:20:04 +00002404 int hlen, tlen;
Ben Greear3bdc0eb2012-02-11 15:39:30 +00002405 int extra_len = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002406
2407 /*
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002408 * Get and verify the address.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002409 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002410
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01002411 if (likely(saddr == NULL)) {
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002412 dev = packet_cached_dev_get(po);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002413 proto = po->num;
2414 addr = NULL;
2415 } else {
2416 err = -EINVAL;
2417 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2418 goto out;
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07002419 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2420 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002421 proto = saddr->sll_protocol;
2422 addr = saddr->sll_addr;
Ben Greear827d9782011-06-01 07:18:53 +00002423 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002424 }
2425
Linus Torvalds1da177e2005-04-16 15:20:36 -07002426 err = -ENXIO;
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002427 if (unlikely(dev == NULL))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002428 goto out_unlock;
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002429 err = -ENETDOWN;
2430 if (unlikely(!(dev->flags & IFF_UP)))
2431 goto out_unlock;
2432
Linus Torvalds1da177e2005-04-16 15:20:36 -07002433 if (sock->type == SOCK_RAW)
2434 reserve = dev->hard_header_len;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002435 if (po->has_vnet_hdr) {
2436 vnet_hdr_len = sizeof(vnet_hdr);
2437
2438 err = -EINVAL;
2439 if (len < vnet_hdr_len)
2440 goto out_unlock;
2441
2442 len -= vnet_hdr_len;
2443
2444 err = memcpy_fromiovec((void *)&vnet_hdr, msg->msg_iov,
2445 vnet_hdr_len);
2446 if (err < 0)
2447 goto out_unlock;
2448
2449 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
2450 (vnet_hdr.csum_start + vnet_hdr.csum_offset + 2 >
2451 vnet_hdr.hdr_len))
2452 vnet_hdr.hdr_len = vnet_hdr.csum_start +
2453 vnet_hdr.csum_offset + 2;
2454
2455 err = -EINVAL;
2456 if (vnet_hdr.hdr_len > len)
2457 goto out_unlock;
2458
2459 if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
2460 switch (vnet_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2461 case VIRTIO_NET_HDR_GSO_TCPV4:
2462 gso_type = SKB_GSO_TCPV4;
2463 break;
2464 case VIRTIO_NET_HDR_GSO_TCPV6:
2465 gso_type = SKB_GSO_TCPV6;
2466 break;
2467 case VIRTIO_NET_HDR_GSO_UDP:
2468 gso_type = SKB_GSO_UDP;
2469 break;
2470 default:
2471 goto out_unlock;
2472 }
2473
2474 if (vnet_hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
2475 gso_type |= SKB_GSO_TCP_ECN;
2476
2477 if (vnet_hdr.gso_size == 0)
2478 goto out_unlock;
2479
2480 }
2481 }
2482
Ben Greear3bdc0eb2012-02-11 15:39:30 +00002483 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
2484 if (!netif_supports_nofcs(dev)) {
2485 err = -EPROTONOSUPPORT;
2486 goto out_unlock;
2487 }
2488 extra_len = 4; /* We're doing our own CRC */
2489 }
2490
Linus Torvalds1da177e2005-04-16 15:20:36 -07002491 err = -EMSGSIZE;
Ben Greear3bdc0eb2012-02-11 15:39:30 +00002492 if (!gso_type && (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002493 goto out_unlock;
2494
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002495 err = -ENOBUFS;
Herbert Xuae641942011-11-18 02:20:04 +00002496 hlen = LL_RESERVED_SPACE(dev);
2497 tlen = dev->needed_tailroom;
2498 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, vnet_hdr.hdr_len,
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002499 msg->msg_flags & MSG_DONTWAIT, &err);
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002500 if (skb == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002501 goto out_unlock;
2502
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002503 skb_set_network_header(skb, reserve);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002504
Stephen Hemminger0c4e8582007-10-09 01:36:32 -07002505 err = -EINVAL;
2506 if (sock->type == SOCK_DGRAM &&
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002507 (offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len)) < 0)
Stephen Hemminger0c4e8582007-10-09 01:36:32 -07002508 goto out_free;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002509
2510 /* Returns -EFAULT on error */
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002511 err = skb_copy_datagram_from_iovec(skb, offset, msg->msg_iov, 0, len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002512 if (err)
2513 goto out_free;
Daniel Borkmannbf84a012013-04-14 08:08:13 +00002514
2515 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002516
Ben Greear3bdc0eb2012-02-11 15:39:30 +00002517 if (!gso_type && (len > dev->mtu + reserve + extra_len)) {
David S. Miller09effa62013-08-07 17:11:00 -07002518 /* Earlier code assumed this would be a VLAN pkt,
2519 * double-check this now that we have the actual
2520 * packet in hand.
2521 */
2522 struct ethhdr *ehdr;
2523 skb_reset_mac_header(skb);
2524 ehdr = eth_hdr(skb);
2525 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
2526 err = -EMSGSIZE;
2527 goto out_free;
2528 }
Ben Greear57f89bf2011-02-11 09:35:18 +00002529 }
2530
David S. Miller09effa62013-08-07 17:11:00 -07002531 skb->protocol = proto;
2532 skb->dev = dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002533 skb->priority = sk->sk_priority;
Eric Dumazet2d37a182009-10-01 19:14:46 +00002534 skb->mark = sk->sk_mark;
Daniel Borkmann0fd5d572014-02-16 15:55:22 +01002535
2536 packet_pick_tx_queue(dev, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002537
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002538 if (po->has_vnet_hdr) {
2539 if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
2540 if (!skb_partial_csum_set(skb, vnet_hdr.csum_start,
2541 vnet_hdr.csum_offset)) {
2542 err = -EINVAL;
2543 goto out_free;
2544 }
2545 }
2546
2547 skb_shinfo(skb)->gso_size = vnet_hdr.gso_size;
2548 skb_shinfo(skb)->gso_type = gso_type;
2549
2550 /* Header must be checked, and gso_segs computed. */
2551 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2552 skb_shinfo(skb)->gso_segs = 0;
2553
2554 len += vnet_hdr_len;
2555 }
2556
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01002557 if (!packet_use_direct_xmit(po))
2558 skb_probe_transport_header(skb, reserve);
Ben Greear3bdc0eb2012-02-11 15:39:30 +00002559 if (unlikely(extra_len == 4))
2560 skb->no_fcs = 1;
2561
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01002562 err = po->xmit(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002563 if (err > 0 && (err = net_xmit_errno(err)) != 0)
2564 goto out_unlock;
2565
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002566 dev_put(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002567
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002568 return len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002569
2570out_free:
2571 kfree_skb(skb);
2572out_unlock:
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002573 if (dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002574 dev_put(dev);
2575out:
2576 return err;
2577}
2578
Johann Baudy69e3c752009-05-18 22:11:22 -07002579static int packet_sendmsg(struct kiocb *iocb, struct socket *sock,
2580 struct msghdr *msg, size_t len)
2581{
Johann Baudy69e3c752009-05-18 22:11:22 -07002582 struct sock *sk = sock->sk;
2583 struct packet_sock *po = pkt_sk(sk);
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01002584
Johann Baudy69e3c752009-05-18 22:11:22 -07002585 if (po->tx_ring.pg_vec)
2586 return tpacket_snd(po, msg);
2587 else
Johann Baudy69e3c752009-05-18 22:11:22 -07002588 return packet_snd(sock, msg, len);
2589}
2590
Linus Torvalds1da177e2005-04-16 15:20:36 -07002591/*
2592 * Close a PACKET socket. This is fairly simple. We immediately go
2593 * to 'closed' state and remove our protocol entry in the device list.
2594 */
2595
2596static int packet_release(struct socket *sock)
2597{
2598 struct sock *sk = sock->sk;
2599 struct packet_sock *po;
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08002600 struct net *net;
chetan lokef6fb8f102011-08-19 10:18:16 +00002601 union tpacket_req_u req_u;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002602
2603 if (!sk)
2604 return 0;
2605
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09002606 net = sock_net(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002607 po = pkt_sk(sk);
2608
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00002609 mutex_lock(&net->packet.sklist_lock);
stephen hemminger808f5112010-02-22 07:57:18 +00002610 sk_del_node_init_rcu(sk);
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00002611 mutex_unlock(&net->packet.sklist_lock);
2612
2613 preempt_disable();
Eric Dumazet920de802008-11-24 00:09:29 -08002614 sock_prot_inuse_add(net, sk->sk_prot, -1);
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00002615 preempt_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002616
stephen hemminger808f5112010-02-22 07:57:18 +00002617 spin_lock(&po->bind_lock);
David S. Millerce06b032011-07-04 01:44:29 -07002618 unregister_prot_hook(sk, false);
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01002619 packet_cached_dev_reset(po);
2620
Ben Greear160ff182011-06-01 07:18:52 +00002621 if (po->prot_hook.dev) {
2622 dev_put(po->prot_hook.dev);
2623 po->prot_hook.dev = NULL;
2624 }
stephen hemminger808f5112010-02-22 07:57:18 +00002625 spin_unlock(&po->bind_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002626
Linus Torvalds1da177e2005-04-16 15:20:36 -07002627 packet_flush_mclist(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002628
Phil Sutter9665d5d2013-02-01 07:21:41 +00002629 if (po->rx_ring.pg_vec) {
2630 memset(&req_u, 0, sizeof(req_u));
chetan lokef6fb8f102011-08-19 10:18:16 +00002631 packet_set_ring(sk, &req_u, 1, 0);
Phil Sutter9665d5d2013-02-01 07:21:41 +00002632 }
Johann Baudy69e3c752009-05-18 22:11:22 -07002633
Phil Sutter9665d5d2013-02-01 07:21:41 +00002634 if (po->tx_ring.pg_vec) {
2635 memset(&req_u, 0, sizeof(req_u));
chetan lokef6fb8f102011-08-19 10:18:16 +00002636 packet_set_ring(sk, &req_u, 1, 1);
Phil Sutter9665d5d2013-02-01 07:21:41 +00002637 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002638
David S. Millerdc99f602011-07-05 01:45:05 -07002639 fanout_release(sk);
2640
stephen hemminger808f5112010-02-22 07:57:18 +00002641 synchronize_net();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002642 /*
2643 * Now the socket is dead. No more input will appear.
2644 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002645 sock_orphan(sk);
2646 sock->sk = NULL;
2647
2648 /* Purge queues */
2649
2650 skb_queue_purge(&sk->sk_receive_queue);
Daniel Borkmannb0138402014-01-15 16:25:36 +01002651 packet_free_pending(po);
Pavel Emelyanov17ab56a2007-11-10 21:38:48 -08002652 sk_refcnt_debug_release(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002653
2654 sock_put(sk);
2655 return 0;
2656}
2657
2658/*
2659 * Attach a packet hook.
2660 */
2661
Daniel Borkmann902fefb2014-01-15 16:25:34 +01002662static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 proto)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002663{
2664 struct packet_sock *po = pkt_sk(sk);
Daniel Borkmann902fefb2014-01-15 16:25:34 +01002665 const struct net_device *dev_curr;
2666 __be16 proto_curr;
2667 bool need_rehook;
David S. Millerdc99f602011-07-05 01:45:05 -07002668
Wei Yongjunaef950b2011-12-27 22:32:41 -05002669 if (po->fanout) {
2670 if (dev)
2671 dev_put(dev);
2672
David S. Millerdc99f602011-07-05 01:45:05 -07002673 return -EINVAL;
Wei Yongjunaef950b2011-12-27 22:32:41 -05002674 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002675
2676 lock_sock(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002677 spin_lock(&po->bind_lock);
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01002678
Daniel Borkmann902fefb2014-01-15 16:25:34 +01002679 proto_curr = po->prot_hook.type;
2680 dev_curr = po->prot_hook.dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002681
Daniel Borkmann902fefb2014-01-15 16:25:34 +01002682 need_rehook = proto_curr != proto || dev_curr != dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002683
Daniel Borkmann902fefb2014-01-15 16:25:34 +01002684 if (need_rehook) {
2685 unregister_prot_hook(sk, true);
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01002686
Daniel Borkmann902fefb2014-01-15 16:25:34 +01002687 po->num = proto;
2688 po->prot_hook.type = proto;
2689
2690 if (po->prot_hook.dev)
2691 dev_put(po->prot_hook.dev);
2692
2693 po->prot_hook.dev = dev;
2694
2695 po->ifindex = dev ? dev->ifindex : 0;
2696 packet_cached_dev_assign(po, dev);
2697 }
2698
2699 if (proto == 0 || !need_rehook)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002700 goto out_unlock;
2701
Urs Thuermannbe85d4a2007-11-12 21:05:20 -08002702 if (!dev || (dev->flags & IFF_UP)) {
David S. Millerce06b032011-07-04 01:44:29 -07002703 register_prot_hook(sk);
Urs Thuermannbe85d4a2007-11-12 21:05:20 -08002704 } else {
2705 sk->sk_err = ENETDOWN;
2706 if (!sock_flag(sk, SOCK_DEAD))
2707 sk->sk_error_report(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002708 }
2709
2710out_unlock:
2711 spin_unlock(&po->bind_lock);
2712 release_sock(sk);
2713 return 0;
2714}
2715
2716/*
2717 * Bind a packet socket to a device
2718 */
2719
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002720static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
2721 int addr_len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002722{
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002723 struct sock *sk = sock->sk;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002724 char name[15];
2725 struct net_device *dev;
2726 int err = -ENODEV;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002727
Linus Torvalds1da177e2005-04-16 15:20:36 -07002728 /*
2729 * Check legality
2730 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002731
Kris Katterjohn8ae55f02006-01-23 16:28:02 -08002732 if (addr_len != sizeof(struct sockaddr))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002733 return -EINVAL;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002734 strlcpy(name, uaddr->sa_data, sizeof(name));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002735
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09002736 dev = dev_get_by_name(sock_net(sk), name);
Ben Greear160ff182011-06-01 07:18:52 +00002737 if (dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002738 err = packet_do_bind(sk, dev, pkt_sk(sk)->num);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002739 return err;
2740}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002741
2742static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
2743{
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002744 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
2745 struct sock *sk = sock->sk;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002746 struct net_device *dev = NULL;
2747 int err;
2748
2749
2750 /*
2751 * Check legality
2752 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002753
Linus Torvalds1da177e2005-04-16 15:20:36 -07002754 if (addr_len < sizeof(struct sockaddr_ll))
2755 return -EINVAL;
2756 if (sll->sll_family != AF_PACKET)
2757 return -EINVAL;
2758
2759 if (sll->sll_ifindex) {
2760 err = -ENODEV;
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09002761 dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002762 if (dev == NULL)
2763 goto out;
2764 }
2765 err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002766
2767out:
2768 return err;
2769}
2770
2771static struct proto packet_proto = {
2772 .name = "PACKET",
2773 .owner = THIS_MODULE,
2774 .obj_size = sizeof(struct packet_sock),
2775};
2776
2777/*
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002778 * Create a packet of type SOCK_PACKET.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002779 */
2780
Eric Paris3f378b62009-11-05 22:18:14 -08002781static int packet_create(struct net *net, struct socket *sock, int protocol,
2782 int kern)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002783{
2784 struct sock *sk;
2785 struct packet_sock *po;
Al Viro0e11c912006-11-08 00:26:29 -08002786 __be16 proto = (__force __be16)protocol; /* weird, but documented */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002787 int err;
2788
Eric W. Biedermandf008c92012-11-16 03:03:07 +00002789 if (!ns_capable(net->user_ns, CAP_NET_RAW))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002790 return -EPERM;
David S. Millerbe020972007-05-29 13:16:31 -07002791 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
2792 sock->type != SOCK_PACKET)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002793 return -ESOCKTNOSUPPORT;
2794
2795 sock->state = SS_UNCONNECTED;
2796
2797 err = -ENOBUFS;
Pavel Emelyanov6257ff22007-11-01 00:39:31 -07002798 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002799 if (sk == NULL)
2800 goto out;
2801
2802 sock->ops = &packet_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002803 if (sock->type == SOCK_PACKET)
2804 sock->ops = &packet_ops_spkt;
David S. Millerbe020972007-05-29 13:16:31 -07002805
Linus Torvalds1da177e2005-04-16 15:20:36 -07002806 sock_init_data(sock, sk);
2807
2808 po = pkt_sk(sk);
2809 sk->sk_family = PF_PACKET;
Al Viro0e11c912006-11-08 00:26:29 -08002810 po->num = proto;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01002811 po->xmit = dev_queue_xmit;
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01002812
Daniel Borkmannb0138402014-01-15 16:25:36 +01002813 err = packet_alloc_pending(po);
2814 if (err)
2815 goto out2;
2816
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01002817 packet_cached_dev_reset(po);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002818
2819 sk->sk_destruct = packet_sock_destruct;
Pavel Emelyanov17ab56a2007-11-10 21:38:48 -08002820 sk_refcnt_debug_inc(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002821
2822 /*
2823 * Attach a protocol block
2824 */
2825
2826 spin_lock_init(&po->bind_lock);
Herbert Xu905db442009-01-30 14:12:06 -08002827 mutex_init(&po->pg_vec_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002828 po->prot_hook.func = packet_rcv;
David S. Millerbe020972007-05-29 13:16:31 -07002829
Linus Torvalds1da177e2005-04-16 15:20:36 -07002830 if (sock->type == SOCK_PACKET)
2831 po->prot_hook.func = packet_rcv_spkt;
David S. Millerbe020972007-05-29 13:16:31 -07002832
Linus Torvalds1da177e2005-04-16 15:20:36 -07002833 po->prot_hook.af_packet_priv = sk;
2834
Al Viro0e11c912006-11-08 00:26:29 -08002835 if (proto) {
2836 po->prot_hook.type = proto;
David S. Millerce06b032011-07-04 01:44:29 -07002837 register_prot_hook(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002838 }
2839
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00002840 mutex_lock(&net->packet.sklist_lock);
stephen hemminger808f5112010-02-22 07:57:18 +00002841 sk_add_node_rcu(sk, &net->packet.sklist);
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00002842 mutex_unlock(&net->packet.sklist_lock);
2843
2844 preempt_disable();
Eric Dumazet36804532008-11-19 14:25:35 -08002845 sock_prot_inuse_add(net, &packet_proto, 1);
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00002846 preempt_enable();
stephen hemminger808f5112010-02-22 07:57:18 +00002847
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002848 return 0;
Daniel Borkmannb0138402014-01-15 16:25:36 +01002849out2:
2850 sk_free(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002851out:
2852 return err;
2853}
2854
2855/*
2856 * Pull a packet from our receive queue and hand it to the user.
2857 * If necessary we block.
2858 */
2859
2860static int packet_recvmsg(struct kiocb *iocb, struct socket *sock,
2861 struct msghdr *msg, size_t len, int flags)
2862{
2863 struct sock *sk = sock->sk;
2864 struct sk_buff *skb;
2865 int copied, err;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002866 int vnet_hdr_len = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002867
2868 err = -EINVAL;
Richard Cochraned85b562010-04-07 22:41:28 +00002869 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002870 goto out;
2871
2872#if 0
2873 /* What error should we return now? EUNATTACH? */
2874 if (pkt_sk(sk)->ifindex < 0)
2875 return -ENODEV;
2876#endif
2877
Richard Cochraned85b562010-04-07 22:41:28 +00002878 if (flags & MSG_ERRQUEUE) {
Richard Cochrancb820f82013-07-19 19:40:09 +02002879 err = sock_recv_errqueue(sk, msg, len,
2880 SOL_PACKET, PACKET_TX_TIMESTAMP);
Richard Cochraned85b562010-04-07 22:41:28 +00002881 goto out;
2882 }
2883
Linus Torvalds1da177e2005-04-16 15:20:36 -07002884 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -07002885 * Call the generic datagram receiver. This handles all sorts
2886 * of horrible races and re-entrancy so we can forget about it
2887 * in the protocol layers.
2888 *
2889 * Now it will return ENETDOWN, if device have just gone down,
2890 * but then it will block.
2891 */
2892
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002893 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002894
2895 /*
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002896 * An error occurred so return it. Because skb_recv_datagram()
Linus Torvalds1da177e2005-04-16 15:20:36 -07002897 * handles the blocking we don't see and worry about blocking
2898 * retries.
2899 */
2900
Kris Katterjohn8ae55f02006-01-23 16:28:02 -08002901 if (skb == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002902 goto out;
2903
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002904 if (pkt_sk(sk)->has_vnet_hdr) {
2905 struct virtio_net_hdr vnet_hdr = { 0 };
2906
2907 err = -EINVAL;
2908 vnet_hdr_len = sizeof(vnet_hdr);
Mariusz Kozlowski1f18b712010-11-08 11:58:45 +00002909 if (len < vnet_hdr_len)
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002910 goto out_free;
2911
Mariusz Kozlowski1f18b712010-11-08 11:58:45 +00002912 len -= vnet_hdr_len;
2913
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002914 if (skb_is_gso(skb)) {
2915 struct skb_shared_info *sinfo = skb_shinfo(skb);
2916
2917 /* This is a hint as to how much should be linear. */
2918 vnet_hdr.hdr_len = skb_headlen(skb);
2919 vnet_hdr.gso_size = sinfo->gso_size;
2920 if (sinfo->gso_type & SKB_GSO_TCPV4)
2921 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
2922 else if (sinfo->gso_type & SKB_GSO_TCPV6)
2923 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
2924 else if (sinfo->gso_type & SKB_GSO_UDP)
2925 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
2926 else if (sinfo->gso_type & SKB_GSO_FCOE)
2927 goto out_free;
2928 else
2929 BUG();
2930 if (sinfo->gso_type & SKB_GSO_TCP_ECN)
2931 vnet_hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
2932 } else
2933 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
2934
2935 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2936 vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
Michał Mirosław55508d62010-12-14 15:24:08 +00002937 vnet_hdr.csum_start = skb_checksum_start_offset(skb);
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002938 vnet_hdr.csum_offset = skb->csum_offset;
Jason Wang10a8d942011-06-10 00:56:17 +00002939 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
2940 vnet_hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002941 } /* else everything is zero */
2942
2943 err = memcpy_toiovec(msg->msg_iov, (void *)&vnet_hdr,
2944 vnet_hdr_len);
2945 if (err < 0)
2946 goto out_free;
2947 }
2948
Hannes Frederic Sowaf3d33422013-11-21 03:14:22 +01002949 /* You lose any data beyond the buffer you gave. If it worries
2950 * a user program they can ask the device for its MTU
2951 * anyway.
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07002952 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002953 copied = skb->len;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002954 if (copied > len) {
2955 copied = len;
2956 msg->msg_flags |= MSG_TRUNC;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002957 }
2958
2959 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
2960 if (err)
2961 goto out_free;
2962
Neil Horman3b885782009-10-12 13:26:31 -07002963 sock_recv_ts_and_drops(msg, sk, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002964
Hannes Frederic Sowaf3d33422013-11-21 03:14:22 +01002965 if (msg->msg_name) {
2966 /* If the address length field is there to be filled
2967 * in, we fill it in now.
2968 */
2969 if (sock->type == SOCK_PACKET) {
Steffen Hurrle342dfc32014-01-17 22:53:15 +01002970 __sockaddr_check_size(sizeof(struct sockaddr_pkt));
Hannes Frederic Sowaf3d33422013-11-21 03:14:22 +01002971 msg->msg_namelen = sizeof(struct sockaddr_pkt);
2972 } else {
2973 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
2974 msg->msg_namelen = sll->sll_halen +
2975 offsetof(struct sockaddr_ll, sll_addr);
2976 }
Herbert Xuffbc6112007-02-04 23:33:10 -08002977 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
2978 msg->msg_namelen);
Hannes Frederic Sowaf3d33422013-11-21 03:14:22 +01002979 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002980
Herbert Xu8dc41942007-02-04 23:31:32 -08002981 if (pkt_sk(sk)->auxdata) {
Herbert Xuffbc6112007-02-04 23:33:10 -08002982 struct tpacket_auxdata aux;
2983
2984 aux.tp_status = TP_STATUS_USER;
2985 if (skb->ip_summed == CHECKSUM_PARTIAL)
2986 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
2987 aux.tp_len = PACKET_SKB_CB(skb)->origlen;
2988 aux.tp_snaplen = skb->len;
2989 aux.tp_mac = 0;
Arnaldo Carvalho de Melobbe735e2007-03-10 22:16:10 -03002990 aux.tp_net = skb_network_offset(skb);
Ben Greeara3bcc232011-06-01 06:49:10 +00002991 if (vlan_tx_tag_present(skb)) {
2992 aux.tp_vlan_tci = vlan_tx_tag_get(skb);
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +09002993 aux.tp_vlan_tpid = ntohs(skb->vlan_proto);
2994 aux.tp_status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
Ben Greeara3bcc232011-06-01 06:49:10 +00002995 } else {
2996 aux.tp_vlan_tci = 0;
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +09002997 aux.tp_vlan_tpid = 0;
Ben Greeara3bcc232011-06-01 06:49:10 +00002998 }
Herbert Xuffbc6112007-02-04 23:33:10 -08002999 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
Herbert Xu8dc41942007-02-04 23:31:32 -08003000 }
3001
Linus Torvalds1da177e2005-04-16 15:20:36 -07003002 /*
3003 * Free or return the buffer as appropriate. Again this
3004 * hides all the races and re-entrancy issues from us.
3005 */
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003006 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003007
3008out_free:
3009 skb_free_datagram(sk, skb);
3010out:
3011 return err;
3012}
3013
Linus Torvalds1da177e2005-04-16 15:20:36 -07003014static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
3015 int *uaddr_len, int peer)
3016{
3017 struct net_device *dev;
3018 struct sock *sk = sock->sk;
3019
3020 if (peer)
3021 return -EOPNOTSUPP;
3022
3023 uaddr->sa_family = AF_PACKET;
Daniel Borkmann2dc85bf2013-06-12 16:02:27 +02003024 memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data));
Eric Dumazet654d1f82009-11-02 10:43:32 +01003025 rcu_read_lock();
3026 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
3027 if (dev)
Daniel Borkmann2dc85bf2013-06-12 16:02:27 +02003028 strlcpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data));
Eric Dumazet654d1f82009-11-02 10:43:32 +01003029 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003030 *uaddr_len = sizeof(*uaddr);
3031
3032 return 0;
3033}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003034
3035static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
3036 int *uaddr_len, int peer)
3037{
3038 struct net_device *dev;
3039 struct sock *sk = sock->sk;
3040 struct packet_sock *po = pkt_sk(sk);
Cyrill Gorcunov13cfa972009-11-08 05:51:19 +00003041 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003042
3043 if (peer)
3044 return -EOPNOTSUPP;
3045
3046 sll->sll_family = AF_PACKET;
3047 sll->sll_ifindex = po->ifindex;
3048 sll->sll_protocol = po->num;
Vasiliy Kulikov67286642010-11-10 12:09:10 -08003049 sll->sll_pkttype = 0;
Eric Dumazet654d1f82009-11-02 10:43:32 +01003050 rcu_read_lock();
3051 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003052 if (dev) {
3053 sll->sll_hatype = dev->type;
3054 sll->sll_halen = dev->addr_len;
3055 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003056 } else {
3057 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
3058 sll->sll_halen = 0;
3059 }
Eric Dumazet654d1f82009-11-02 10:43:32 +01003060 rcu_read_unlock();
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07003061 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003062
3063 return 0;
3064}
3065
Wang Chen2aeb0b82008-07-14 20:49:46 -07003066static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
3067 int what)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003068{
3069 switch (i->type) {
3070 case PACKET_MR_MULTICAST:
Jiri Pirko11625632010-03-02 20:40:01 +00003071 if (i->alen != dev->addr_len)
3072 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003073 if (what > 0)
Jiri Pirko22bedad32010-04-01 21:22:57 +00003074 return dev_mc_add(dev, i->addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003075 else
Jiri Pirko22bedad32010-04-01 21:22:57 +00003076 return dev_mc_del(dev, i->addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003077 break;
3078 case PACKET_MR_PROMISC:
Wang Chen2aeb0b82008-07-14 20:49:46 -07003079 return dev_set_promiscuity(dev, what);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003080 case PACKET_MR_ALLMULTI:
Wang Chen2aeb0b82008-07-14 20:49:46 -07003081 return dev_set_allmulti(dev, what);
Eric W. Biedermand95ed922009-05-19 18:27:17 +00003082 case PACKET_MR_UNICAST:
Jiri Pirko11625632010-03-02 20:40:01 +00003083 if (i->alen != dev->addr_len)
3084 return -EINVAL;
Eric W. Biedermand95ed922009-05-19 18:27:17 +00003085 if (what > 0)
Jiri Pirkoa748ee22010-04-01 21:22:09 +00003086 return dev_uc_add(dev, i->addr);
Eric W. Biedermand95ed922009-05-19 18:27:17 +00003087 else
Jiri Pirkoa748ee22010-04-01 21:22:09 +00003088 return dev_uc_del(dev, i->addr);
Eric W. Biedermand95ed922009-05-19 18:27:17 +00003089 break;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003090 default:
3091 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003092 }
Wang Chen2aeb0b82008-07-14 20:49:46 -07003093 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003094}
3095
3096static void packet_dev_mclist(struct net_device *dev, struct packet_mclist *i, int what)
3097{
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003098 for ( ; i; i = i->next) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003099 if (i->ifindex == dev->ifindex)
3100 packet_dev_mc(dev, i, what);
3101 }
3102}
3103
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07003104static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003105{
3106 struct packet_sock *po = pkt_sk(sk);
3107 struct packet_mclist *ml, *i;
3108 struct net_device *dev;
3109 int err;
3110
3111 rtnl_lock();
3112
3113 err = -ENODEV;
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09003114 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003115 if (!dev)
3116 goto done;
3117
3118 err = -EINVAL;
Jiri Pirko11625632010-03-02 20:40:01 +00003119 if (mreq->mr_alen > dev->addr_len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003120 goto done;
3121
3122 err = -ENOBUFS;
Kris Katterjohn8b3a7002006-01-11 15:56:43 -08003123 i = kmalloc(sizeof(*i), GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003124 if (i == NULL)
3125 goto done;
3126
3127 err = 0;
3128 for (ml = po->mclist; ml; ml = ml->next) {
3129 if (ml->ifindex == mreq->mr_ifindex &&
3130 ml->type == mreq->mr_type &&
3131 ml->alen == mreq->mr_alen &&
3132 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3133 ml->count++;
3134 /* Free the new element ... */
3135 kfree(i);
3136 goto done;
3137 }
3138 }
3139
3140 i->type = mreq->mr_type;
3141 i->ifindex = mreq->mr_ifindex;
3142 i->alen = mreq->mr_alen;
3143 memcpy(i->addr, mreq->mr_address, i->alen);
3144 i->count = 1;
3145 i->next = po->mclist;
3146 po->mclist = i;
Wang Chen2aeb0b82008-07-14 20:49:46 -07003147 err = packet_dev_mc(dev, i, 1);
3148 if (err) {
3149 po->mclist = i->next;
3150 kfree(i);
3151 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003152
3153done:
3154 rtnl_unlock();
3155 return err;
3156}
3157
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07003158static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003159{
3160 struct packet_mclist *ml, **mlp;
3161
3162 rtnl_lock();
3163
3164 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
3165 if (ml->ifindex == mreq->mr_ifindex &&
3166 ml->type == mreq->mr_type &&
3167 ml->alen == mreq->mr_alen &&
3168 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3169 if (--ml->count == 0) {
3170 struct net_device *dev;
3171 *mlp = ml->next;
Eric Dumazetad959e72009-10-16 06:38:46 +00003172 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3173 if (dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003174 packet_dev_mc(dev, ml, -1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003175 kfree(ml);
3176 }
3177 rtnl_unlock();
3178 return 0;
3179 }
3180 }
3181 rtnl_unlock();
3182 return -EADDRNOTAVAIL;
3183}
3184
3185static void packet_flush_mclist(struct sock *sk)
3186{
3187 struct packet_sock *po = pkt_sk(sk);
3188 struct packet_mclist *ml;
3189
3190 if (!po->mclist)
3191 return;
3192
3193 rtnl_lock();
3194 while ((ml = po->mclist) != NULL) {
3195 struct net_device *dev;
3196
3197 po->mclist = ml->next;
Eric Dumazetad959e72009-10-16 06:38:46 +00003198 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3199 if (dev != NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003200 packet_dev_mc(dev, ml, -1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003201 kfree(ml);
3202 }
3203 rtnl_unlock();
3204}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003205
3206static int
David S. Millerb7058842009-09-30 16:12:20 -07003207packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003208{
3209 struct sock *sk = sock->sk;
Herbert Xu8dc41942007-02-04 23:31:32 -08003210 struct packet_sock *po = pkt_sk(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003211 int ret;
3212
3213 if (level != SOL_PACKET)
3214 return -ENOPROTOOPT;
3215
Johann Baudy69e3c752009-05-18 22:11:22 -07003216 switch (optname) {
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003217 case PACKET_ADD_MEMBERSHIP:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003218 case PACKET_DROP_MEMBERSHIP:
3219 {
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07003220 struct packet_mreq_max mreq;
3221 int len = optlen;
3222 memset(&mreq, 0, sizeof(mreq));
3223 if (len < sizeof(struct packet_mreq))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003224 return -EINVAL;
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07003225 if (len > sizeof(mreq))
3226 len = sizeof(mreq);
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003227 if (copy_from_user(&mreq, optval, len))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003228 return -EFAULT;
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07003229 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3230 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003231 if (optname == PACKET_ADD_MEMBERSHIP)
3232 ret = packet_mc_add(sk, &mreq);
3233 else
3234 ret = packet_mc_drop(sk, &mreq);
3235 return ret;
3236 }
David S. Millera2efcfa2007-05-29 13:12:50 -07003237
Linus Torvalds1da177e2005-04-16 15:20:36 -07003238 case PACKET_RX_RING:
Johann Baudy69e3c752009-05-18 22:11:22 -07003239 case PACKET_TX_RING:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003240 {
chetan lokef6fb8f102011-08-19 10:18:16 +00003241 union tpacket_req_u req_u;
3242 int len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003243
chetan lokef6fb8f102011-08-19 10:18:16 +00003244 switch (po->tp_version) {
3245 case TPACKET_V1:
3246 case TPACKET_V2:
3247 len = sizeof(req_u.req);
3248 break;
3249 case TPACKET_V3:
3250 default:
3251 len = sizeof(req_u.req3);
3252 break;
3253 }
3254 if (optlen < len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003255 return -EINVAL;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003256 if (pkt_sk(sk)->has_vnet_hdr)
3257 return -EINVAL;
chetan lokef6fb8f102011-08-19 10:18:16 +00003258 if (copy_from_user(&req_u.req, optval, len))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003259 return -EFAULT;
chetan lokef6fb8f102011-08-19 10:18:16 +00003260 return packet_set_ring(sk, &req_u, 0,
3261 optname == PACKET_TX_RING);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003262 }
3263 case PACKET_COPY_THRESH:
3264 {
3265 int val;
3266
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003267 if (optlen != sizeof(val))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003268 return -EINVAL;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003269 if (copy_from_user(&val, optval, sizeof(val)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003270 return -EFAULT;
3271
3272 pkt_sk(sk)->copy_thresh = val;
3273 return 0;
3274 }
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003275 case PACKET_VERSION:
3276 {
3277 int val;
3278
3279 if (optlen != sizeof(val))
3280 return -EINVAL;
Johann Baudy69e3c752009-05-18 22:11:22 -07003281 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003282 return -EBUSY;
3283 if (copy_from_user(&val, optval, sizeof(val)))
3284 return -EFAULT;
3285 switch (val) {
3286 case TPACKET_V1:
3287 case TPACKET_V2:
chetan lokef6fb8f102011-08-19 10:18:16 +00003288 case TPACKET_V3:
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003289 po->tp_version = val;
3290 return 0;
3291 default:
3292 return -EINVAL;
3293 }
3294 }
Patrick McHardy8913336a2008-07-18 18:05:19 -07003295 case PACKET_RESERVE:
3296 {
3297 unsigned int val;
3298
3299 if (optlen != sizeof(val))
3300 return -EINVAL;
Johann Baudy69e3c752009-05-18 22:11:22 -07003301 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
Patrick McHardy8913336a2008-07-18 18:05:19 -07003302 return -EBUSY;
3303 if (copy_from_user(&val, optval, sizeof(val)))
3304 return -EFAULT;
3305 po->tp_reserve = val;
3306 return 0;
3307 }
Johann Baudy69e3c752009-05-18 22:11:22 -07003308 case PACKET_LOSS:
3309 {
3310 unsigned int val;
3311
3312 if (optlen != sizeof(val))
3313 return -EINVAL;
3314 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3315 return -EBUSY;
3316 if (copy_from_user(&val, optval, sizeof(val)))
3317 return -EFAULT;
3318 po->tp_loss = !!val;
3319 return 0;
3320 }
Herbert Xu8dc41942007-02-04 23:31:32 -08003321 case PACKET_AUXDATA:
3322 {
3323 int val;
3324
3325 if (optlen < sizeof(val))
3326 return -EINVAL;
3327 if (copy_from_user(&val, optval, sizeof(val)))
3328 return -EFAULT;
3329
3330 po->auxdata = !!val;
3331 return 0;
3332 }
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07003333 case PACKET_ORIGDEV:
3334 {
3335 int val;
3336
3337 if (optlen < sizeof(val))
3338 return -EINVAL;
3339 if (copy_from_user(&val, optval, sizeof(val)))
3340 return -EFAULT;
3341
3342 po->origdev = !!val;
3343 return 0;
3344 }
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003345 case PACKET_VNET_HDR:
3346 {
3347 int val;
3348
3349 if (sock->type != SOCK_RAW)
3350 return -EINVAL;
3351 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3352 return -EBUSY;
3353 if (optlen < sizeof(val))
3354 return -EINVAL;
3355 if (copy_from_user(&val, optval, sizeof(val)))
3356 return -EFAULT;
3357
3358 po->has_vnet_hdr = !!val;
3359 return 0;
3360 }
Scott McMillan614f60f2010-06-02 05:53:56 -07003361 case PACKET_TIMESTAMP:
3362 {
3363 int val;
3364
3365 if (optlen != sizeof(val))
3366 return -EINVAL;
3367 if (copy_from_user(&val, optval, sizeof(val)))
3368 return -EFAULT;
3369
3370 po->tp_tstamp = val;
3371 return 0;
3372 }
David S. Millerdc99f602011-07-05 01:45:05 -07003373 case PACKET_FANOUT:
3374 {
3375 int val;
3376
3377 if (optlen != sizeof(val))
3378 return -EINVAL;
3379 if (copy_from_user(&val, optval, sizeof(val)))
3380 return -EFAULT;
3381
3382 return fanout_add(sk, val & 0xffff, val >> 16);
3383 }
Paul Chavent5920cd3a2012-11-06 23:10:47 +00003384 case PACKET_TX_HAS_OFF:
3385 {
3386 unsigned int val;
3387
3388 if (optlen != sizeof(val))
3389 return -EINVAL;
3390 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3391 return -EBUSY;
3392 if (copy_from_user(&val, optval, sizeof(val)))
3393 return -EFAULT;
3394 po->tp_tx_has_off = !!val;
3395 return 0;
3396 }
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01003397 case PACKET_QDISC_BYPASS:
3398 {
3399 int val;
3400
3401 if (optlen != sizeof(val))
3402 return -EINVAL;
3403 if (copy_from_user(&val, optval, sizeof(val)))
3404 return -EFAULT;
3405
3406 po->xmit = val ? packet_direct_xmit : dev_queue_xmit;
3407 return 0;
3408 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003409 default:
3410 return -ENOPROTOOPT;
3411 }
3412}
3413
3414static int packet_getsockopt(struct socket *sock, int level, int optname,
3415 char __user *optval, int __user *optlen)
3416{
3417 int len;
Eric Dumazetc06fff62012-04-19 21:56:11 +00003418 int val, lv = sizeof(val);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003419 struct sock *sk = sock->sk;
3420 struct packet_sock *po = pkt_sk(sk);
Eric Dumazetc06fff62012-04-19 21:56:11 +00003421 void *data = &val;
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00003422 union tpacket_stats_u st;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003423
3424 if (level != SOL_PACKET)
3425 return -ENOPROTOOPT;
3426
Kris Katterjohn8ae55f02006-01-23 16:28:02 -08003427 if (get_user(len, optlen))
3428 return -EFAULT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003429
3430 if (len < 0)
3431 return -EINVAL;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003432
Johann Baudy69e3c752009-05-18 22:11:22 -07003433 switch (optname) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003434 case PACKET_STATISTICS:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003435 spin_lock_bh(&sk->sk_receive_queue.lock);
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00003436 memcpy(&st, &po->stats, sizeof(st));
3437 memset(&po->stats, 0, sizeof(po->stats));
3438 spin_unlock_bh(&sk->sk_receive_queue.lock);
3439
chetan lokef6fb8f102011-08-19 10:18:16 +00003440 if (po->tp_version == TPACKET_V3) {
Eric Dumazetc06fff62012-04-19 21:56:11 +00003441 lv = sizeof(struct tpacket_stats_v3);
Willem de Bruijn8bcdeaf2013-08-19 16:40:22 -04003442 st.stats3.tp_packets += st.stats3.tp_drops;
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00003443 data = &st.stats3;
chetan lokef6fb8f102011-08-19 10:18:16 +00003444 } else {
Eric Dumazetc06fff62012-04-19 21:56:11 +00003445 lv = sizeof(struct tpacket_stats);
Willem de Bruijn8bcdeaf2013-08-19 16:40:22 -04003446 st.stats1.tp_packets += st.stats1.tp_drops;
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00003447 data = &st.stats1;
chetan lokef6fb8f102011-08-19 10:18:16 +00003448 }
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00003449
Linus Torvalds1da177e2005-04-16 15:20:36 -07003450 break;
Herbert Xu8dc41942007-02-04 23:31:32 -08003451 case PACKET_AUXDATA:
Herbert Xu8dc41942007-02-04 23:31:32 -08003452 val = po->auxdata;
Herbert Xu8dc41942007-02-04 23:31:32 -08003453 break;
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07003454 case PACKET_ORIGDEV:
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07003455 val = po->origdev;
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07003456 break;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003457 case PACKET_VNET_HDR:
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003458 val = po->has_vnet_hdr;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003459 break;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003460 case PACKET_VERSION:
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003461 val = po->tp_version;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003462 break;
3463 case PACKET_HDRLEN:
3464 if (len > sizeof(int))
3465 len = sizeof(int);
3466 if (copy_from_user(&val, optval, len))
3467 return -EFAULT;
3468 switch (val) {
3469 case TPACKET_V1:
3470 val = sizeof(struct tpacket_hdr);
3471 break;
3472 case TPACKET_V2:
3473 val = sizeof(struct tpacket2_hdr);
3474 break;
chetan lokef6fb8f102011-08-19 10:18:16 +00003475 case TPACKET_V3:
3476 val = sizeof(struct tpacket3_hdr);
3477 break;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003478 default:
3479 return -EINVAL;
3480 }
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003481 break;
Patrick McHardy8913336a2008-07-18 18:05:19 -07003482 case PACKET_RESERVE:
Patrick McHardy8913336a2008-07-18 18:05:19 -07003483 val = po->tp_reserve;
Patrick McHardy8913336a2008-07-18 18:05:19 -07003484 break;
Johann Baudy69e3c752009-05-18 22:11:22 -07003485 case PACKET_LOSS:
Johann Baudy69e3c752009-05-18 22:11:22 -07003486 val = po->tp_loss;
Johann Baudy69e3c752009-05-18 22:11:22 -07003487 break;
Scott McMillan614f60f2010-06-02 05:53:56 -07003488 case PACKET_TIMESTAMP:
Scott McMillan614f60f2010-06-02 05:53:56 -07003489 val = po->tp_tstamp;
Scott McMillan614f60f2010-06-02 05:53:56 -07003490 break;
David S. Millerdc99f602011-07-05 01:45:05 -07003491 case PACKET_FANOUT:
David S. Millerdc99f602011-07-05 01:45:05 -07003492 val = (po->fanout ?
3493 ((u32)po->fanout->id |
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00003494 ((u32)po->fanout->type << 16) |
3495 ((u32)po->fanout->flags << 24)) :
David S. Millerdc99f602011-07-05 01:45:05 -07003496 0);
David S. Millerdc99f602011-07-05 01:45:05 -07003497 break;
Paul Chavent5920cd3a2012-11-06 23:10:47 +00003498 case PACKET_TX_HAS_OFF:
3499 val = po->tp_tx_has_off;
3500 break;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01003501 case PACKET_QDISC_BYPASS:
3502 val = packet_use_direct_xmit(po);
3503 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003504 default:
3505 return -ENOPROTOOPT;
3506 }
3507
Eric Dumazetc06fff62012-04-19 21:56:11 +00003508 if (len > lv)
3509 len = lv;
Kris Katterjohn8ae55f02006-01-23 16:28:02 -08003510 if (put_user(len, optlen))
3511 return -EFAULT;
Herbert Xu8dc41942007-02-04 23:31:32 -08003512 if (copy_to_user(optval, data, len))
3513 return -EFAULT;
Kris Katterjohn8ae55f02006-01-23 16:28:02 -08003514 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003515}
3516
3517
Jiri Pirko351638e2013-05-28 01:30:21 +00003518static int packet_notifier(struct notifier_block *this,
3519 unsigned long msg, void *ptr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003520{
3521 struct sock *sk;
Jiri Pirko351638e2013-05-28 01:30:21 +00003522 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09003523 struct net *net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003524
stephen hemminger808f5112010-02-22 07:57:18 +00003525 rcu_read_lock();
Sasha Levinb67bfe02013-02-27 17:06:00 -08003526 sk_for_each_rcu(sk, &net->packet.sklist) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003527 struct packet_sock *po = pkt_sk(sk);
3528
3529 switch (msg) {
3530 case NETDEV_UNREGISTER:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003531 if (po->mclist)
3532 packet_dev_mclist(dev, po->mclist, -1);
David S. Millera2efcfa2007-05-29 13:12:50 -07003533 /* fallthrough */
3534
Linus Torvalds1da177e2005-04-16 15:20:36 -07003535 case NETDEV_DOWN:
3536 if (dev->ifindex == po->ifindex) {
3537 spin_lock(&po->bind_lock);
3538 if (po->running) {
David S. Millerce06b032011-07-04 01:44:29 -07003539 __unregister_prot_hook(sk, false);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003540 sk->sk_err = ENETDOWN;
3541 if (!sock_flag(sk, SOCK_DEAD))
3542 sk->sk_error_report(sk);
3543 }
3544 if (msg == NETDEV_UNREGISTER) {
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01003545 packet_cached_dev_reset(po);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003546 po->ifindex = -1;
Ben Greear160ff182011-06-01 07:18:52 +00003547 if (po->prot_hook.dev)
3548 dev_put(po->prot_hook.dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003549 po->prot_hook.dev = NULL;
3550 }
3551 spin_unlock(&po->bind_lock);
3552 }
3553 break;
3554 case NETDEV_UP:
stephen hemminger808f5112010-02-22 07:57:18 +00003555 if (dev->ifindex == po->ifindex) {
3556 spin_lock(&po->bind_lock);
David S. Millerce06b032011-07-04 01:44:29 -07003557 if (po->num)
3558 register_prot_hook(sk);
stephen hemminger808f5112010-02-22 07:57:18 +00003559 spin_unlock(&po->bind_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003560 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003561 break;
3562 }
3563 }
stephen hemminger808f5112010-02-22 07:57:18 +00003564 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003565 return NOTIFY_DONE;
3566}
3567
3568
3569static int packet_ioctl(struct socket *sock, unsigned int cmd,
3570 unsigned long arg)
3571{
3572 struct sock *sk = sock->sk;
3573
Johann Baudy69e3c752009-05-18 22:11:22 -07003574 switch (cmd) {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003575 case SIOCOUTQ:
3576 {
3577 int amount = sk_wmem_alloc_get(sk);
Eric Dumazet31e6d362009-06-17 19:05:41 -07003578
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003579 return put_user(amount, (int __user *)arg);
3580 }
3581 case SIOCINQ:
3582 {
3583 struct sk_buff *skb;
3584 int amount = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003585
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003586 spin_lock_bh(&sk->sk_receive_queue.lock);
3587 skb = skb_peek(&sk->sk_receive_queue);
3588 if (skb)
3589 amount = skb->len;
3590 spin_unlock_bh(&sk->sk_receive_queue.lock);
3591 return put_user(amount, (int __user *)arg);
3592 }
3593 case SIOCGSTAMP:
3594 return sock_get_timestamp(sk, (struct timeval __user *)arg);
3595 case SIOCGSTAMPNS:
3596 return sock_get_timestampns(sk, (struct timespec __user *)arg);
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003597
Linus Torvalds1da177e2005-04-16 15:20:36 -07003598#ifdef CONFIG_INET
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003599 case SIOCADDRT:
3600 case SIOCDELRT:
3601 case SIOCDARP:
3602 case SIOCGARP:
3603 case SIOCSARP:
3604 case SIOCGIFADDR:
3605 case SIOCSIFADDR:
3606 case SIOCGIFBRDADDR:
3607 case SIOCSIFBRDADDR:
3608 case SIOCGIFNETMASK:
3609 case SIOCSIFNETMASK:
3610 case SIOCGIFDSTADDR:
3611 case SIOCSIFDSTADDR:
3612 case SIOCSIFFLAGS:
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003613 return inet_dgram_ops.ioctl(sock, cmd, arg);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003614#endif
3615
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003616 default:
3617 return -ENOIOCTLCMD;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003618 }
3619 return 0;
3620}
3621
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003622static unsigned int packet_poll(struct file *file, struct socket *sock,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003623 poll_table *wait)
3624{
3625 struct sock *sk = sock->sk;
3626 struct packet_sock *po = pkt_sk(sk);
3627 unsigned int mask = datagram_poll(file, sock, wait);
3628
3629 spin_lock_bh(&sk->sk_receive_queue.lock);
Johann Baudy69e3c752009-05-18 22:11:22 -07003630 if (po->rx_ring.pg_vec) {
chetan lokef6fb8f102011-08-19 10:18:16 +00003631 if (!packet_previous_rx_frame(po, &po->rx_ring,
3632 TP_STATUS_KERNEL))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003633 mask |= POLLIN | POLLRDNORM;
3634 }
3635 spin_unlock_bh(&sk->sk_receive_queue.lock);
Johann Baudy69e3c752009-05-18 22:11:22 -07003636 spin_lock_bh(&sk->sk_write_queue.lock);
3637 if (po->tx_ring.pg_vec) {
3638 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
3639 mask |= POLLOUT | POLLWRNORM;
3640 }
3641 spin_unlock_bh(&sk->sk_write_queue.lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003642 return mask;
3643}
3644
3645
3646/* Dirty? Well, I still did not learn better way to account
3647 * for user mmaps.
3648 */
3649
3650static void packet_mm_open(struct vm_area_struct *vma)
3651{
3652 struct file *file = vma->vm_file;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003653 struct socket *sock = file->private_data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003654 struct sock *sk = sock->sk;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003655
Linus Torvalds1da177e2005-04-16 15:20:36 -07003656 if (sk)
3657 atomic_inc(&pkt_sk(sk)->mapped);
3658}
3659
3660static void packet_mm_close(struct vm_area_struct *vma)
3661{
3662 struct file *file = vma->vm_file;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003663 struct socket *sock = file->private_data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003664 struct sock *sk = sock->sk;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003665
Linus Torvalds1da177e2005-04-16 15:20:36 -07003666 if (sk)
3667 atomic_dec(&pkt_sk(sk)->mapped);
3668}
3669
Alexey Dobriyanf0f37e22009-09-27 22:29:37 +04003670static const struct vm_operations_struct packet_mmap_ops = {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003671 .open = packet_mm_open,
3672 .close = packet_mm_close,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003673};
3674
Neil Horman0e3125c2010-11-16 10:26:47 -08003675static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
3676 unsigned int len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003677{
3678 int i;
3679
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003680 for (i = 0; i < len; i++) {
Neil Horman0e3125c2010-11-16 10:26:47 -08003681 if (likely(pg_vec[i].buffer)) {
Changli Gaoc56b4d92010-12-01 02:52:57 +00003682 if (is_vmalloc_addr(pg_vec[i].buffer))
Neil Horman0e3125c2010-11-16 10:26:47 -08003683 vfree(pg_vec[i].buffer);
3684 else
3685 free_pages((unsigned long)pg_vec[i].buffer,
3686 order);
3687 pg_vec[i].buffer = NULL;
3688 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003689 }
3690 kfree(pg_vec);
3691}
3692
Olof Johanssoneea49cc92011-11-02 11:00:49 +00003693static char *alloc_one_pg_vec_page(unsigned long order)
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003694{
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01003695 char *buffer;
Neil Horman0e3125c2010-11-16 10:26:47 -08003696 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
3697 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
Eric Dumazet719bfea2009-04-15 03:39:52 -07003698
Neil Horman0e3125c2010-11-16 10:26:47 -08003699 buffer = (char *) __get_free_pages(gfp_flags, order);
Neil Horman0e3125c2010-11-16 10:26:47 -08003700 if (buffer)
3701 return buffer;
3702
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01003703 /* __get_free_pages failed, fall back to vmalloc */
Eric Dumazetbbce5a52010-11-20 07:31:54 +00003704 buffer = vzalloc((1 << order) * PAGE_SIZE);
Neil Horman0e3125c2010-11-16 10:26:47 -08003705 if (buffer)
3706 return buffer;
3707
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01003708 /* vmalloc failed, lets dig into swap here */
Neil Horman0e3125c2010-11-16 10:26:47 -08003709 gfp_flags &= ~__GFP_NORETRY;
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01003710 buffer = (char *) __get_free_pages(gfp_flags, order);
Neil Horman0e3125c2010-11-16 10:26:47 -08003711 if (buffer)
3712 return buffer;
3713
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01003714 /* complete and utter failure */
Neil Horman0e3125c2010-11-16 10:26:47 -08003715 return NULL;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003716}
3717
Neil Horman0e3125c2010-11-16 10:26:47 -08003718static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003719{
3720 unsigned int block_nr = req->tp_block_nr;
Neil Horman0e3125c2010-11-16 10:26:47 -08003721 struct pgv *pg_vec;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003722 int i;
3723
Neil Horman0e3125c2010-11-16 10:26:47 -08003724 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003725 if (unlikely(!pg_vec))
3726 goto out;
3727
3728 for (i = 0; i < block_nr; i++) {
Changli Gaoc56b4d92010-12-01 02:52:57 +00003729 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
Neil Horman0e3125c2010-11-16 10:26:47 -08003730 if (unlikely(!pg_vec[i].buffer))
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003731 goto out_free_pgvec;
3732 }
3733
3734out:
3735 return pg_vec;
3736
3737out_free_pgvec:
3738 free_pg_vec(pg_vec, order, block_nr);
3739 pg_vec = NULL;
3740 goto out;
3741}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003742
chetan lokef6fb8f102011-08-19 10:18:16 +00003743static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
Johann Baudy69e3c752009-05-18 22:11:22 -07003744 int closing, int tx_ring)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003745{
Neil Horman0e3125c2010-11-16 10:26:47 -08003746 struct pgv *pg_vec = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003747 struct packet_sock *po = pkt_sk(sk);
Al Viro0e11c912006-11-08 00:26:29 -08003748 int was_running, order = 0;
Johann Baudy69e3c752009-05-18 22:11:22 -07003749 struct packet_ring_buffer *rb;
3750 struct sk_buff_head *rb_queue;
Al Viro0e11c912006-11-08 00:26:29 -08003751 __be16 num;
chetan lokef6fb8f102011-08-19 10:18:16 +00003752 int err = -EINVAL;
3753 /* Added to avoid minimal code churn */
3754 struct tpacket_req *req = &req_u->req;
3755
3756 /* Opening a Tx-ring is NOT supported in TPACKET_V3 */
3757 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) {
3758 WARN(1, "Tx-ring is not supported.\n");
3759 goto out;
3760 }
Johann Baudy69e3c752009-05-18 22:11:22 -07003761
3762 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
3763 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
3764
3765 err = -EBUSY;
3766 if (!closing) {
3767 if (atomic_read(&po->mapped))
3768 goto out;
Daniel Borkmannb0138402014-01-15 16:25:36 +01003769 if (packet_read_pending(rb))
Johann Baudy69e3c752009-05-18 22:11:22 -07003770 goto out;
3771 }
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003772
Linus Torvalds1da177e2005-04-16 15:20:36 -07003773 if (req->tp_block_nr) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003774 /* Sanity tests and some calculations */
Johann Baudy69e3c752009-05-18 22:11:22 -07003775 err = -EBUSY;
3776 if (unlikely(rb->pg_vec))
3777 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003778
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003779 switch (po->tp_version) {
3780 case TPACKET_V1:
3781 po->tp_hdrlen = TPACKET_HDRLEN;
3782 break;
3783 case TPACKET_V2:
3784 po->tp_hdrlen = TPACKET2_HDRLEN;
3785 break;
chetan lokef6fb8f102011-08-19 10:18:16 +00003786 case TPACKET_V3:
3787 po->tp_hdrlen = TPACKET3_HDRLEN;
3788 break;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003789 }
3790
Johann Baudy69e3c752009-05-18 22:11:22 -07003791 err = -EINVAL;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003792 if (unlikely((int)req->tp_block_size <= 0))
Johann Baudy69e3c752009-05-18 22:11:22 -07003793 goto out;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003794 if (unlikely(req->tp_block_size & (PAGE_SIZE - 1)))
Johann Baudy69e3c752009-05-18 22:11:22 -07003795 goto out;
Eric Dumazetdc808112014-08-15 09:16:04 -07003796 if (po->tp_version >= TPACKET_V3 &&
3797 (int)(req->tp_block_size -
3798 BLK_PLUS_PRIV(req_u->req3.tp_sizeof_priv)) <= 0)
3799 goto out;
Patrick McHardy8913336a2008-07-18 18:05:19 -07003800 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
Johann Baudy69e3c752009-05-18 22:11:22 -07003801 po->tp_reserve))
3802 goto out;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003803 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
Johann Baudy69e3c752009-05-18 22:11:22 -07003804 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003805
Johann Baudy69e3c752009-05-18 22:11:22 -07003806 rb->frames_per_block = req->tp_block_size/req->tp_frame_size;
3807 if (unlikely(rb->frames_per_block <= 0))
3808 goto out;
3809 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
3810 req->tp_frame_nr))
3811 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003812
3813 err = -ENOMEM;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003814 order = get_order(req->tp_block_size);
3815 pg_vec = alloc_pg_vec(req, order);
3816 if (unlikely(!pg_vec))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003817 goto out;
chetan lokef6fb8f102011-08-19 10:18:16 +00003818 switch (po->tp_version) {
3819 case TPACKET_V3:
3820 /* Transmit path is not supported. We checked
3821 * it above but just being paranoid
3822 */
3823 if (!tx_ring)
3824 init_prb_bdqc(po, rb, pg_vec, req_u, tx_ring);
Dan Carpenterd7cf0c32014-02-18 15:20:51 +03003825 break;
chetan lokef6fb8f102011-08-19 10:18:16 +00003826 default:
3827 break;
3828 }
Johann Baudy69e3c752009-05-18 22:11:22 -07003829 }
3830 /* Done */
3831 else {
3832 err = -EINVAL;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003833 if (unlikely(req->tp_frame_nr))
Johann Baudy69e3c752009-05-18 22:11:22 -07003834 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003835 }
3836
3837 lock_sock(sk);
3838
3839 /* Detach socket from network */
3840 spin_lock(&po->bind_lock);
3841 was_running = po->running;
3842 num = po->num;
3843 if (was_running) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003844 po->num = 0;
David S. Millerce06b032011-07-04 01:44:29 -07003845 __unregister_prot_hook(sk, false);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003846 }
3847 spin_unlock(&po->bind_lock);
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003848
Linus Torvalds1da177e2005-04-16 15:20:36 -07003849 synchronize_net();
3850
3851 err = -EBUSY;
Herbert Xu905db442009-01-30 14:12:06 -08003852 mutex_lock(&po->pg_vec_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003853 if (closing || atomic_read(&po->mapped) == 0) {
3854 err = 0;
Johann Baudy69e3c752009-05-18 22:11:22 -07003855 spin_lock_bh(&rb_queue->lock);
Changli Gaoc053fd92010-12-10 16:02:20 -08003856 swap(rb->pg_vec, pg_vec);
Johann Baudy69e3c752009-05-18 22:11:22 -07003857 rb->frame_max = (req->tp_frame_nr - 1);
3858 rb->head = 0;
3859 rb->frame_size = req->tp_frame_size;
3860 spin_unlock_bh(&rb_queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003861
Changli Gaoc053fd92010-12-10 16:02:20 -08003862 swap(rb->pg_vec_order, order);
3863 swap(rb->pg_vec_len, req->tp_block_nr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003864
Johann Baudy69e3c752009-05-18 22:11:22 -07003865 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
3866 po->prot_hook.func = (po->rx_ring.pg_vec) ?
3867 tpacket_rcv : packet_rcv;
3868 skb_queue_purge(rb_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003869 if (atomic_read(&po->mapped))
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003870 pr_err("packet_mmap: vma is busy: %d\n",
3871 atomic_read(&po->mapped));
Linus Torvalds1da177e2005-04-16 15:20:36 -07003872 }
Herbert Xu905db442009-01-30 14:12:06 -08003873 mutex_unlock(&po->pg_vec_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003874
3875 spin_lock(&po->bind_lock);
David S. Millerce06b032011-07-04 01:44:29 -07003876 if (was_running) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003877 po->num = num;
David S. Millerce06b032011-07-04 01:44:29 -07003878 register_prot_hook(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003879 }
3880 spin_unlock(&po->bind_lock);
chetan lokef6fb8f102011-08-19 10:18:16 +00003881 if (closing && (po->tp_version > TPACKET_V2)) {
3882 /* Because we don't support block-based V3 on tx-ring */
3883 if (!tx_ring)
3884 prb_shutdown_retire_blk_timer(po, tx_ring, rb_queue);
3885 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003886 release_sock(sk);
3887
Linus Torvalds1da177e2005-04-16 15:20:36 -07003888 if (pg_vec)
3889 free_pg_vec(pg_vec, order, req->tp_block_nr);
3890out:
3891 return err;
3892}
3893
Johann Baudy69e3c752009-05-18 22:11:22 -07003894static int packet_mmap(struct file *file, struct socket *sock,
3895 struct vm_area_struct *vma)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003896{
3897 struct sock *sk = sock->sk;
3898 struct packet_sock *po = pkt_sk(sk);
Johann Baudy69e3c752009-05-18 22:11:22 -07003899 unsigned long size, expected_size;
3900 struct packet_ring_buffer *rb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003901 unsigned long start;
3902 int err = -EINVAL;
3903 int i;
3904
3905 if (vma->vm_pgoff)
3906 return -EINVAL;
3907
Herbert Xu905db442009-01-30 14:12:06 -08003908 mutex_lock(&po->pg_vec_lock);
Johann Baudy69e3c752009-05-18 22:11:22 -07003909
3910 expected_size = 0;
3911 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3912 if (rb->pg_vec) {
3913 expected_size += rb->pg_vec_len
3914 * rb->pg_vec_pages
3915 * PAGE_SIZE;
3916 }
3917 }
3918
3919 if (expected_size == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003920 goto out;
Johann Baudy69e3c752009-05-18 22:11:22 -07003921
3922 size = vma->vm_end - vma->vm_start;
3923 if (size != expected_size)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003924 goto out;
3925
Linus Torvalds1da177e2005-04-16 15:20:36 -07003926 start = vma->vm_start;
Johann Baudy69e3c752009-05-18 22:11:22 -07003927 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3928 if (rb->pg_vec == NULL)
3929 continue;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003930
Johann Baudy69e3c752009-05-18 22:11:22 -07003931 for (i = 0; i < rb->pg_vec_len; i++) {
Neil Horman0e3125c2010-11-16 10:26:47 -08003932 struct page *page;
3933 void *kaddr = rb->pg_vec[i].buffer;
Johann Baudy69e3c752009-05-18 22:11:22 -07003934 int pg_num;
3935
Changli Gaoc56b4d92010-12-01 02:52:57 +00003936 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
3937 page = pgv_to_page(kaddr);
Johann Baudy69e3c752009-05-18 22:11:22 -07003938 err = vm_insert_page(vma, start, page);
3939 if (unlikely(err))
3940 goto out;
3941 start += PAGE_SIZE;
Neil Horman0e3125c2010-11-16 10:26:47 -08003942 kaddr += PAGE_SIZE;
Johann Baudy69e3c752009-05-18 22:11:22 -07003943 }
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003944 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003945 }
Johann Baudy69e3c752009-05-18 22:11:22 -07003946
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003947 atomic_inc(&po->mapped);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003948 vma->vm_ops = &packet_mmap_ops;
3949 err = 0;
3950
3951out:
Herbert Xu905db442009-01-30 14:12:06 -08003952 mutex_unlock(&po->pg_vec_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003953 return err;
3954}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003955
Eric Dumazet90ddc4f2005-12-22 12:49:22 -08003956static const struct proto_ops packet_ops_spkt = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003957 .family = PF_PACKET,
3958 .owner = THIS_MODULE,
3959 .release = packet_release,
3960 .bind = packet_bind_spkt,
3961 .connect = sock_no_connect,
3962 .socketpair = sock_no_socketpair,
3963 .accept = sock_no_accept,
3964 .getname = packet_getname_spkt,
3965 .poll = datagram_poll,
3966 .ioctl = packet_ioctl,
3967 .listen = sock_no_listen,
3968 .shutdown = sock_no_shutdown,
3969 .setsockopt = sock_no_setsockopt,
3970 .getsockopt = sock_no_getsockopt,
3971 .sendmsg = packet_sendmsg_spkt,
3972 .recvmsg = packet_recvmsg,
3973 .mmap = sock_no_mmap,
3974 .sendpage = sock_no_sendpage,
3975};
Linus Torvalds1da177e2005-04-16 15:20:36 -07003976
Eric Dumazet90ddc4f2005-12-22 12:49:22 -08003977static const struct proto_ops packet_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003978 .family = PF_PACKET,
3979 .owner = THIS_MODULE,
3980 .release = packet_release,
3981 .bind = packet_bind,
3982 .connect = sock_no_connect,
3983 .socketpair = sock_no_socketpair,
3984 .accept = sock_no_accept,
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003985 .getname = packet_getname,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003986 .poll = packet_poll,
3987 .ioctl = packet_ioctl,
3988 .listen = sock_no_listen,
3989 .shutdown = sock_no_shutdown,
3990 .setsockopt = packet_setsockopt,
3991 .getsockopt = packet_getsockopt,
3992 .sendmsg = packet_sendmsg,
3993 .recvmsg = packet_recvmsg,
3994 .mmap = packet_mmap,
3995 .sendpage = sock_no_sendpage,
3996};
3997
Stephen Hemmingerec1b4cf2009-10-05 05:58:39 +00003998static const struct net_proto_family packet_family_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003999 .family = PF_PACKET,
4000 .create = packet_create,
4001 .owner = THIS_MODULE,
4002};
4003
4004static struct notifier_block packet_netdev_notifier = {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00004005 .notifier_call = packet_notifier,
Linus Torvalds1da177e2005-04-16 15:20:36 -07004006};
4007
4008#ifdef CONFIG_PROC_FS
Linus Torvalds1da177e2005-04-16 15:20:36 -07004009
4010static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
stephen hemminger808f5112010-02-22 07:57:18 +00004011 __acquires(RCU)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004012{
Denis V. Luneve372c412007-11-19 22:31:54 -08004013 struct net *net = seq_file_net(seq);
stephen hemminger808f5112010-02-22 07:57:18 +00004014
4015 rcu_read_lock();
4016 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004017}
4018
4019static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4020{
Herbert Xu1bf40952007-12-16 14:04:02 -08004021 struct net *net = seq_file_net(seq);
stephen hemminger808f5112010-02-22 07:57:18 +00004022 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004023}
4024
4025static void packet_seq_stop(struct seq_file *seq, void *v)
stephen hemminger808f5112010-02-22 07:57:18 +00004026 __releases(RCU)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004027{
stephen hemminger808f5112010-02-22 07:57:18 +00004028 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004029}
4030
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09004031static int packet_seq_show(struct seq_file *seq, void *v)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004032{
4033 if (v == SEQ_START_TOKEN)
4034 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
4035 else {
Li Zefanb7ceabd2010-02-08 23:19:29 +00004036 struct sock *s = sk_entry(v);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004037 const struct packet_sock *po = pkt_sk(s);
4038
4039 seq_printf(seq,
Dan Rosenberg71338aa2011-05-23 12:17:35 +00004040 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07004041 s,
4042 atomic_read(&s->sk_refcnt),
4043 s->sk_type,
4044 ntohs(po->num),
4045 po->ifindex,
4046 po->running,
4047 atomic_read(&s->sk_rmem_alloc),
Eric W. Biedermana7cb5a42012-05-24 01:10:10 -06004048 from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00004049 sock_i_ino(s));
Linus Torvalds1da177e2005-04-16 15:20:36 -07004050 }
4051
4052 return 0;
4053}
4054
Philippe De Muyter56b3d972007-07-10 23:07:31 -07004055static const struct seq_operations packet_seq_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004056 .start = packet_seq_start,
4057 .next = packet_seq_next,
4058 .stop = packet_seq_stop,
4059 .show = packet_seq_show,
4060};
4061
4062static int packet_seq_open(struct inode *inode, struct file *file)
4063{
Denis V. Luneve372c412007-11-19 22:31:54 -08004064 return seq_open_net(inode, file, &packet_seq_ops,
4065 sizeof(struct seq_net_private));
Linus Torvalds1da177e2005-04-16 15:20:36 -07004066}
4067
Arjan van de Venda7071d2007-02-12 00:55:36 -08004068static const struct file_operations packet_seq_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004069 .owner = THIS_MODULE,
4070 .open = packet_seq_open,
4071 .read = seq_read,
4072 .llseek = seq_lseek,
Denis V. Luneve372c412007-11-19 22:31:54 -08004073 .release = seq_release_net,
Linus Torvalds1da177e2005-04-16 15:20:36 -07004074};
4075
4076#endif
4077
Alexey Dobriyan2c8c1e72010-01-17 03:35:32 +00004078static int __net_init packet_net_init(struct net *net)
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004079{
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00004080 mutex_init(&net->packet.sklist_lock);
Denis V. Lunev2aaef4e2007-12-11 04:19:54 -08004081 INIT_HLIST_HEAD(&net->packet.sklist);
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004082
Gao fengd4beaa62013-02-18 01:34:54 +00004083 if (!proc_create("packet", 0, net->proc_net, &packet_seq_fops))
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004084 return -ENOMEM;
4085
4086 return 0;
4087}
4088
Alexey Dobriyan2c8c1e72010-01-17 03:35:32 +00004089static void __net_exit packet_net_exit(struct net *net)
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004090{
Gao fengece31ff2013-02-18 01:34:56 +00004091 remove_proc_entry("packet", net->proc_net);
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004092}
4093
4094static struct pernet_operations packet_net_ops = {
4095 .init = packet_net_init,
4096 .exit = packet_net_exit,
4097};
4098
4099
Linus Torvalds1da177e2005-04-16 15:20:36 -07004100static void __exit packet_exit(void)
4101{
Linus Torvalds1da177e2005-04-16 15:20:36 -07004102 unregister_netdevice_notifier(&packet_netdev_notifier);
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004103 unregister_pernet_subsys(&packet_net_ops);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004104 sock_unregister(PF_PACKET);
4105 proto_unregister(&packet_proto);
4106}
4107
4108static int __init packet_init(void)
4109{
4110 int rc = proto_register(&packet_proto, 0);
4111
4112 if (rc != 0)
4113 goto out;
4114
4115 sock_register(&packet_family_ops);
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004116 register_pernet_subsys(&packet_net_ops);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004117 register_netdevice_notifier(&packet_netdev_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004118out:
4119 return rc;
4120}
4121
4122module_init(packet_init);
4123module_exit(packet_exit);
4124MODULE_LICENSE("GPL");
4125MODULE_ALIAS_NETPROTO(PF_PACKET);