blob: 6a563e6e24dee83783c328adf7b7b3927fd3572f [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * PACKET - implements raw packet sockets.
7 *
Jesper Juhl02c30a82005-05-05 16:16:16 -07008 * Authors: Ross Biro
Linus Torvalds1da177e2005-04-16 15:20:36 -07009 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
11 *
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +090012 * Fixes:
Linus Torvalds1da177e2005-04-16 15:20:36 -070013 * Alan Cox : verify_area() now used correctly
14 * Alan Cox : new skbuff lists, look ma no backlogs!
15 * Alan Cox : tidied skbuff lists.
16 * Alan Cox : Now uses generic datagram routines I
17 * added. Also fixed the peek/read crash
18 * from all old Linux datagram code.
19 * Alan Cox : Uses the improved datagram code.
20 * Alan Cox : Added NULL's for socket options.
21 * Alan Cox : Re-commented the code.
22 * Alan Cox : Use new kernel side addressing
23 * Rob Janssen : Correct MTU usage.
24 * Dave Platt : Counter leaks caused by incorrect
25 * interrupt locking and some slightly
26 * dubious gcc output. Can you read
27 * compiler: it said _VOLATILE_
28 * Richard Kooijman : Timestamp fixes.
29 * Alan Cox : New buffers. Use sk->mac.raw.
30 * Alan Cox : sendmsg/recvmsg support.
31 * Alan Cox : Protocol setting support
32 * Alexey Kuznetsov : Untied from IPv4 stack.
33 * Cyrus Durgin : Fixed kerneld for kmod.
34 * Michal Ostrowski : Module initialization cleanup.
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +090035 * Ulises Alonso : Frame number limit removal and
Linus Torvalds1da177e2005-04-16 15:20:36 -070036 * packet_set_ring memory leak.
Eric W. Biederman0fb375f2005-09-21 00:11:37 -070037 * Eric Biederman : Allow for > 8 byte hardware addresses.
38 * The convention is that longer addresses
39 * will simply extend the hardware address
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +090040 * byte arrays at the end of sockaddr_ll
Eric W. Biederman0fb375f2005-09-21 00:11:37 -070041 * and packet_mreq.
Johann Baudy69e3c752009-05-18 22:11:22 -070042 * Johann Baudy : Added TX RING.
chetan lokef6fb8f102011-08-19 10:18:16 +000043 * Chetan Loke : Implemented TPACKET_V3 block abstraction
44 * layer.
45 * Copyright (C) 2011, <lokec@ccs.neu.edu>
46 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070047 *
48 * This program is free software; you can redistribute it and/or
49 * modify it under the terms of the GNU General Public License
50 * as published by the Free Software Foundation; either version
51 * 2 of the License, or (at your option) any later version.
52 *
53 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +090054
Linus Torvalds1da177e2005-04-16 15:20:36 -070055#include <linux/types.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070056#include <linux/mm.h>
Randy Dunlap4fc268d2006-01-11 12:17:47 -080057#include <linux/capability.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070058#include <linux/fcntl.h>
59#include <linux/socket.h>
60#include <linux/in.h>
61#include <linux/inet.h>
62#include <linux/netdevice.h>
63#include <linux/if_packet.h>
64#include <linux/wireless.h>
Herbert Xuffbc6112007-02-04 23:33:10 -080065#include <linux/kernel.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070066#include <linux/kmod.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090067#include <linux/slab.h>
Neil Horman0e3125c2010-11-16 10:26:47 -080068#include <linux/vmalloc.h>
Eric W. Biederman457c4cb2007-09-12 12:01:34 +020069#include <net/net_namespace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070070#include <net/ip.h>
71#include <net/protocol.h>
72#include <linux/skbuff.h>
73#include <net/sock.h>
74#include <linux/errno.h>
75#include <linux/timer.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070076#include <asm/uaccess.h>
77#include <asm/ioctls.h>
78#include <asm/page.h>
Al Viroa1f8e7f72006-10-19 16:08:53 -040079#include <asm/cacheflush.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070080#include <asm/io.h>
81#include <linux/proc_fs.h>
82#include <linux/seq_file.h>
83#include <linux/poll.h>
84#include <linux/module.h>
85#include <linux/init.h>
Herbert Xu905db442009-01-30 14:12:06 -080086#include <linux/mutex.h>
Eric Dumazet05423b22009-10-26 18:40:35 -070087#include <linux/if_vlan.h>
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -080088#include <linux/virtio_net.h>
Richard Cochraned85b562010-04-07 22:41:28 +000089#include <linux/errqueue.h>
Scott McMillan614f60f2010-06-02 05:53:56 -070090#include <linux/net_tstamp.h>
Daniel Borkmannb0138402014-01-15 16:25:36 +010091#include <linux/percpu.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070092#ifdef CONFIG_INET
93#include <net/inet_common.h>
94#endif
Willem de Bruijn47dceb82015-08-14 22:31:34 -040095#include <linux/bpf.h>
Willem de Bruijn719c44d2016-06-07 12:06:34 -040096#include <net/compat.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070097
Pavel Emelyanov2787b042012-08-13 05:49:39 +000098#include "internal.h"
99
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101 Assumptions:
102 - if device has no dev->hard_header routine, it adds and removes ll header
103 inside itself. In this case ll header is invisible outside of device,
104 but higher levels still should reserve dev->hard_header_len.
105 Some devices are enough clever to reallocate skb, when header
106 will not fit to reserved space (tunnel), another ones are silly
107 (PPP).
108 - packet socket receives packets with pulled ll header,
109 so that SOCK_RAW should push it back.
110
111On receive:
112-----------
113
114Incoming, dev->hard_header!=NULL
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -0700115 mac_header -> ll header
116 data -> data
Linus Torvalds1da177e2005-04-16 15:20:36 -0700117
118Outgoing, dev->hard_header!=NULL
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -0700119 mac_header -> ll header
120 data -> ll header
Linus Torvalds1da177e2005-04-16 15:20:36 -0700121
122Incoming, dev->hard_header==NULL
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -0700123 mac_header -> UNKNOWN position. It is very likely, that it points to ll
124 header. PPP makes it, that is wrong, because introduce
YOSHIFUJI Hideakidb0c58f2007-07-19 10:44:35 +0900125 assymetry between rx and tx paths.
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -0700126 data -> data
Linus Torvalds1da177e2005-04-16 15:20:36 -0700127
128Outgoing, dev->hard_header==NULL
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -0700129 mac_header -> data. ll header is still not built!
130 data -> data
Linus Torvalds1da177e2005-04-16 15:20:36 -0700131
132Resume
133 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
134
135
136On transmit:
137------------
138
139dev->hard_header != NULL
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -0700140 mac_header -> ll header
141 data -> ll header
Linus Torvalds1da177e2005-04-16 15:20:36 -0700142
143dev->hard_header == NULL (ll header is added by device, we cannot control it)
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -0700144 mac_header -> data
145 data -> data
Linus Torvalds1da177e2005-04-16 15:20:36 -0700146
147 We should set nh.raw on output to correct posistion,
148 packet classifier depends on it.
149 */
150
Linus Torvalds1da177e2005-04-16 15:20:36 -0700151/* Private packet socket structures. */
152
Eric W. Biederman0fb375f2005-09-21 00:11:37 -0700153/* identical to struct packet_mreq except it has
154 * a longer address field.
155 */
Eric Dumazet40d4e3d2009-07-21 21:57:59 +0000156struct packet_mreq_max {
Eric W. Biederman0fb375f2005-09-21 00:11:37 -0700157 int mr_ifindex;
158 unsigned short mr_type;
159 unsigned short mr_alen;
160 unsigned char mr_address[MAX_ADDR_LEN];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700161};
David S. Millera2efcfa2007-05-29 13:12:50 -0700162
Daniel Borkmann184f4892013-04-16 01:57:46 +0000163union tpacket_uhdr {
164 struct tpacket_hdr *h1;
165 struct tpacket2_hdr *h2;
166 struct tpacket3_hdr *h3;
167 void *raw;
168};
169
chetan lokef6fb8f102011-08-19 10:18:16 +0000170static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
Johann Baudy69e3c752009-05-18 22:11:22 -0700171 int closing, int tx_ring);
172
chetan lokef6fb8f102011-08-19 10:18:16 +0000173#define V3_ALIGNMENT (8)
174
chetan lokebc59ba32011-08-25 10:43:30 +0000175#define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
chetan lokef6fb8f102011-08-19 10:18:16 +0000176
177#define BLK_PLUS_PRIV(sz_of_priv) \
178 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
179
chetan lokef6fb8f102011-08-19 10:18:16 +0000180#define PGV_FROM_VMALLOC 1
Johann Baudy69e3c752009-05-18 22:11:22 -0700181
chetan lokef6fb8f102011-08-19 10:18:16 +0000182#define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
183#define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
184#define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
185#define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
186#define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
187#define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
188#define BLOCK_PRIV(x) ((void *)((char *)(x) + BLOCK_O2PRIV(x)))
189
Johann Baudy69e3c752009-05-18 22:11:22 -0700190struct packet_sock;
191static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
Willem de Bruijn77f65eb2013-03-19 10:18:11 +0000192static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
193 struct packet_type *pt, struct net_device *orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700194
chetan lokef6fb8f102011-08-19 10:18:16 +0000195static void *packet_previous_frame(struct packet_sock *po,
196 struct packet_ring_buffer *rb,
197 int status);
198static void packet_increment_head(struct packet_ring_buffer *buff);
chetan lokebc59ba32011-08-25 10:43:30 +0000199static int prb_curr_blk_in_use(struct tpacket_kbdq_core *,
200 struct tpacket_block_desc *);
201static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
chetan lokef6fb8f102011-08-19 10:18:16 +0000202 struct packet_sock *);
chetan lokebc59ba32011-08-25 10:43:30 +0000203static void prb_retire_current_block(struct tpacket_kbdq_core *,
chetan lokef6fb8f102011-08-19 10:18:16 +0000204 struct packet_sock *, unsigned int status);
chetan lokebc59ba32011-08-25 10:43:30 +0000205static int prb_queue_frozen(struct tpacket_kbdq_core *);
206static void prb_open_block(struct tpacket_kbdq_core *,
207 struct tpacket_block_desc *);
chetan lokef6fb8f102011-08-19 10:18:16 +0000208static void prb_retire_rx_blk_timer_expired(unsigned long);
chetan lokebc59ba32011-08-25 10:43:30 +0000209static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
210static void prb_init_blk_timer(struct packet_sock *,
211 struct tpacket_kbdq_core *,
212 void (*func) (unsigned long));
213static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
214static void prb_clear_rxhash(struct tpacket_kbdq_core *,
215 struct tpacket3_hdr *);
216static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
217 struct tpacket3_hdr *);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700218static void packet_flush_mclist(struct sock *sk);
219
Herbert Xuffbc6112007-02-04 23:33:10 -0800220struct packet_skb_cb {
Herbert Xuffbc6112007-02-04 23:33:10 -0800221 union {
222 struct sockaddr_pkt pkt;
Eyal Birger2472d762015-03-01 14:58:28 +0200223 union {
224 /* Trick: alias skb original length with
225 * ll.sll_family and ll.protocol in order
226 * to save room.
227 */
228 unsigned int origlen;
229 struct sockaddr_ll ll;
230 };
Herbert Xuffbc6112007-02-04 23:33:10 -0800231 } sa;
232};
233
David Woodhoused3869ef2015-09-23 19:45:08 +0100234#define vio_le() virtio_legacy_is_little_endian()
235
Herbert Xuffbc6112007-02-04 23:33:10 -0800236#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
Herbert Xu8dc41942007-02-04 23:31:32 -0800237
chetan lokebc59ba32011-08-25 10:43:30 +0000238#define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
chetan lokef6fb8f102011-08-19 10:18:16 +0000239#define GET_PBLOCK_DESC(x, bid) \
chetan lokebc59ba32011-08-25 10:43:30 +0000240 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
chetan lokef6fb8f102011-08-19 10:18:16 +0000241#define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
chetan lokebc59ba32011-08-25 10:43:30 +0000242 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
chetan lokef6fb8f102011-08-19 10:18:16 +0000243#define GET_NEXT_PRB_BLK_NUM(x) \
244 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
245 ((x)->kactive_blk_num+1) : 0)
246
David S. Millerdc99f602011-07-05 01:45:05 -0700247static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
248static void __fanout_link(struct sock *sk, struct packet_sock *po);
249
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100250static int packet_direct_xmit(struct sk_buff *skb)
251{
252 struct net_device *dev = skb->dev;
Willem de Bruijn104ba782016-10-26 11:23:07 -0400253 struct sk_buff *orig_skb = skb;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100254 struct netdev_queue *txq;
Daniel Borkmann43279502014-03-27 16:38:30 +0100255 int ret = NETDEV_TX_BUSY;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100256
257 if (unlikely(!netif_running(dev) ||
Daniel Borkmann43279502014-03-27 16:38:30 +0100258 !netif_carrier_ok(dev)))
259 goto drop;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100260
Willem de Bruijn104ba782016-10-26 11:23:07 -0400261 skb = validate_xmit_skb_list(skb, dev);
262 if (skb != orig_skb)
Daniel Borkmann43279502014-03-27 16:38:30 +0100263 goto drop;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100264
Daniel Borkmann10c51b56232014-08-27 11:11:27 +0200265 txq = skb_get_tx_queue(dev, skb);
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100266
Daniel Borkmann43279502014-03-27 16:38:30 +0100267 local_bh_disable();
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100268
Daniel Borkmann43279502014-03-27 16:38:30 +0100269 HARD_TX_LOCK(dev, txq, smp_processor_id());
David S. Miller10b3ad82014-08-29 21:07:24 -0700270 if (!netif_xmit_frozen_or_drv_stopped(txq))
David S. Millerfa2dbdc2014-08-29 21:55:22 -0700271 ret = netdev_start_xmit(skb, dev, txq, false);
Daniel Borkmann43279502014-03-27 16:38:30 +0100272 HARD_TX_UNLOCK(dev, txq);
273
274 local_bh_enable();
275
276 if (!dev_xmit_complete(ret))
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100277 kfree_skb(skb);
Daniel Borkmann43279502014-03-27 16:38:30 +0100278
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100279 return ret;
Daniel Borkmann43279502014-03-27 16:38:30 +0100280drop:
Daniel Borkmann0f97ede2014-04-02 20:52:56 +0200281 atomic_long_inc(&dev->tx_dropped);
Willem de Bruijn104ba782016-10-26 11:23:07 -0400282 kfree_skb_list(skb);
Daniel Borkmann43279502014-03-27 16:38:30 +0100283 return NET_XMIT_DROP;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100284}
285
Daniel Borkmann66e56cd2013-12-06 11:36:15 +0100286static struct net_device *packet_cached_dev_get(struct packet_sock *po)
287{
288 struct net_device *dev;
289
290 rcu_read_lock();
291 dev = rcu_dereference(po->cached_dev);
292 if (likely(dev))
293 dev_hold(dev);
294 rcu_read_unlock();
295
296 return dev;
297}
298
299static void packet_cached_dev_assign(struct packet_sock *po,
300 struct net_device *dev)
301{
302 rcu_assign_pointer(po->cached_dev, dev);
303}
304
305static void packet_cached_dev_reset(struct packet_sock *po)
306{
307 RCU_INIT_POINTER(po->cached_dev, NULL);
308}
309
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100310static bool packet_use_direct_xmit(const struct packet_sock *po)
311{
312 return po->xmit == packet_direct_xmit;
313}
314
Daniel Borkmann0fd5d572014-02-16 15:55:22 +0100315static u16 __packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb)
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100316{
Li Zhong1cbac012013-12-12 22:39:55 +0100317 return (u16) raw_smp_processor_id() % dev->real_num_tx_queues;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100318}
319
Daniel Borkmann0fd5d572014-02-16 15:55:22 +0100320static void packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb)
321{
322 const struct net_device_ops *ops = dev->netdev_ops;
323 u16 queue_index;
324
325 if (ops->ndo_select_queue) {
326 queue_index = ops->ndo_select_queue(dev, skb, NULL,
327 __packet_pick_tx_queue);
328 queue_index = netdev_cap_txqueue(dev, queue_index);
329 } else {
330 queue_index = __packet_pick_tx_queue(dev, skb);
331 }
332
333 skb_set_queue_mapping(skb, queue_index);
334}
335
David S. Millerce06b032011-07-04 01:44:29 -0700336/* register_prot_hook must be invoked with the po->bind_lock held,
337 * or from a context in which asynchronous accesses to the packet
338 * socket is not possible (packet_create()).
339 */
340static void register_prot_hook(struct sock *sk)
341{
342 struct packet_sock *po = pkt_sk(sk);
Daniel Borkmanne40526c2013-11-21 16:50:58 +0100343
David S. Millerce06b032011-07-04 01:44:29 -0700344 if (!po->running) {
Daniel Borkmann66e56cd2013-12-06 11:36:15 +0100345 if (po->fanout)
David S. Millerdc99f602011-07-05 01:45:05 -0700346 __fanout_link(sk, po);
Daniel Borkmann66e56cd2013-12-06 11:36:15 +0100347 else
David S. Millerdc99f602011-07-05 01:45:05 -0700348 dev_add_pack(&po->prot_hook);
Daniel Borkmanne40526c2013-11-21 16:50:58 +0100349
David S. Millerce06b032011-07-04 01:44:29 -0700350 sock_hold(sk);
351 po->running = 1;
352 }
353}
354
355/* {,__}unregister_prot_hook() must be invoked with the po->bind_lock
356 * held. If the sync parameter is true, we will temporarily drop
357 * the po->bind_lock and do a synchronize_net to make sure no
358 * asynchronous packet processing paths still refer to the elements
359 * of po->prot_hook. If the sync parameter is false, it is the
360 * callers responsibility to take care of this.
361 */
362static void __unregister_prot_hook(struct sock *sk, bool sync)
363{
364 struct packet_sock *po = pkt_sk(sk);
365
366 po->running = 0;
Daniel Borkmann66e56cd2013-12-06 11:36:15 +0100367
368 if (po->fanout)
David S. Millerdc99f602011-07-05 01:45:05 -0700369 __fanout_unlink(sk, po);
Daniel Borkmann66e56cd2013-12-06 11:36:15 +0100370 else
David S. Millerdc99f602011-07-05 01:45:05 -0700371 __dev_remove_pack(&po->prot_hook);
Daniel Borkmanne40526c2013-11-21 16:50:58 +0100372
David S. Millerce06b032011-07-04 01:44:29 -0700373 __sock_put(sk);
374
375 if (sync) {
376 spin_unlock(&po->bind_lock);
377 synchronize_net();
378 spin_lock(&po->bind_lock);
379 }
380}
381
382static void unregister_prot_hook(struct sock *sk, bool sync)
383{
384 struct packet_sock *po = pkt_sk(sk);
385
386 if (po->running)
387 __unregister_prot_hook(sk, sync);
388}
389
Michael S. Tsirkin6e580402014-11-24 13:32:16 +0200390static inline struct page * __pure pgv_to_page(void *addr)
Changli Gao0af55bb2010-12-01 02:52:20 +0000391{
392 if (is_vmalloc_addr(addr))
393 return vmalloc_to_page(addr);
394 return virt_to_page(addr);
395}
396
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700397static void __packet_set_status(struct packet_sock *po, void *frame, int status)
398{
Daniel Borkmann184f4892013-04-16 01:57:46 +0000399 union tpacket_uhdr h;
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700400
401 h.raw = frame;
402 switch (po->tp_version) {
403 case TPACKET_V1:
404 h.h1->tp_status = status;
Changli Gao0af55bb2010-12-01 02:52:20 +0000405 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700406 break;
407 case TPACKET_V2:
408 h.h2->tp_status = status;
Changli Gao0af55bb2010-12-01 02:52:20 +0000409 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700410 break;
chetan lokef6fb8f102011-08-19 10:18:16 +0000411 case TPACKET_V3:
Johann Baudy69e3c752009-05-18 22:11:22 -0700412 default:
chetan lokef6fb8f102011-08-19 10:18:16 +0000413 WARN(1, "TPACKET version not supported.\n");
Johann Baudy69e3c752009-05-18 22:11:22 -0700414 BUG();
415 }
416
417 smp_wmb();
418}
419
420static int __packet_get_status(struct packet_sock *po, void *frame)
421{
Daniel Borkmann184f4892013-04-16 01:57:46 +0000422 union tpacket_uhdr h;
Johann Baudy69e3c752009-05-18 22:11:22 -0700423
424 smp_rmb();
425
426 h.raw = frame;
427 switch (po->tp_version) {
428 case TPACKET_V1:
Changli Gao0af55bb2010-12-01 02:52:20 +0000429 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
Johann Baudy69e3c752009-05-18 22:11:22 -0700430 return h.h1->tp_status;
431 case TPACKET_V2:
Changli Gao0af55bb2010-12-01 02:52:20 +0000432 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
Johann Baudy69e3c752009-05-18 22:11:22 -0700433 return h.h2->tp_status;
chetan lokef6fb8f102011-08-19 10:18:16 +0000434 case TPACKET_V3:
Johann Baudy69e3c752009-05-18 22:11:22 -0700435 default:
chetan lokef6fb8f102011-08-19 10:18:16 +0000436 WARN(1, "TPACKET version not supported.\n");
Johann Baudy69e3c752009-05-18 22:11:22 -0700437 BUG();
438 return 0;
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700439 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700440}
Johann Baudy69e3c752009-05-18 22:11:22 -0700441
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000442static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec *ts,
443 unsigned int flags)
Daniel Borkmann7a513842013-04-23 00:39:29 +0000444{
445 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
446
Willem de Bruijn68a360e2014-07-25 18:01:31 -0400447 if (shhwtstamps &&
448 (flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
449 ktime_to_timespec_cond(shhwtstamps->hwtstamp, ts))
450 return TP_STATUS_TS_RAW_HARDWARE;
Daniel Borkmann7a513842013-04-23 00:39:29 +0000451
452 if (ktime_to_timespec_cond(skb->tstamp, ts))
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000453 return TP_STATUS_TS_SOFTWARE;
Daniel Borkmann7a513842013-04-23 00:39:29 +0000454
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000455 return 0;
Daniel Borkmann7a513842013-04-23 00:39:29 +0000456}
457
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000458static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
459 struct sk_buff *skb)
Willem de Bruijn2e313962013-04-23 00:39:28 +0000460{
461 union tpacket_uhdr h;
462 struct timespec ts;
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000463 __u32 ts_status;
Willem de Bruijn2e313962013-04-23 00:39:28 +0000464
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000465 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
466 return 0;
Willem de Bruijn2e313962013-04-23 00:39:28 +0000467
468 h.raw = frame;
469 switch (po->tp_version) {
470 case TPACKET_V1:
471 h.h1->tp_sec = ts.tv_sec;
472 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
473 break;
474 case TPACKET_V2:
475 h.h2->tp_sec = ts.tv_sec;
476 h.h2->tp_nsec = ts.tv_nsec;
477 break;
478 case TPACKET_V3:
479 default:
480 WARN(1, "TPACKET version not supported.\n");
481 BUG();
482 }
483
484 /* one flush is safe, as both fields always lie on the same cacheline */
485 flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
486 smp_wmb();
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000487
488 return ts_status;
Willem de Bruijn2e313962013-04-23 00:39:28 +0000489}
490
Johann Baudy69e3c752009-05-18 22:11:22 -0700491static void *packet_lookup_frame(struct packet_sock *po,
492 struct packet_ring_buffer *rb,
493 unsigned int position,
494 int status)
495{
496 unsigned int pg_vec_pos, frame_offset;
Daniel Borkmann184f4892013-04-16 01:57:46 +0000497 union tpacket_uhdr h;
Johann Baudy69e3c752009-05-18 22:11:22 -0700498
499 pg_vec_pos = position / rb->frames_per_block;
500 frame_offset = position % rb->frames_per_block;
501
Neil Horman0e3125c2010-11-16 10:26:47 -0800502 h.raw = rb->pg_vec[pg_vec_pos].buffer +
503 (frame_offset * rb->frame_size);
Johann Baudy69e3c752009-05-18 22:11:22 -0700504
505 if (status != __packet_get_status(po, h.raw))
506 return NULL;
507
508 return h.raw;
509}
510
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000511static void *packet_current_frame(struct packet_sock *po,
Johann Baudy69e3c752009-05-18 22:11:22 -0700512 struct packet_ring_buffer *rb,
513 int status)
514{
515 return packet_lookup_frame(po, rb, rb->head, status);
516}
517
chetan lokebc59ba32011-08-25 10:43:30 +0000518static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
chetan lokef6fb8f102011-08-19 10:18:16 +0000519{
520 del_timer_sync(&pkc->retire_blk_timer);
521}
522
523static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
chetan lokef6fb8f102011-08-19 10:18:16 +0000524 struct sk_buff_head *rb_queue)
525{
chetan lokebc59ba32011-08-25 10:43:30 +0000526 struct tpacket_kbdq_core *pkc;
chetan lokef6fb8f102011-08-19 10:18:16 +0000527
Tobias Klauser73d0fcf2015-07-28 14:21:26 +0200528 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
chetan lokef6fb8f102011-08-19 10:18:16 +0000529
Veaceslav Falicoec6f8092013-11-29 09:53:23 +0100530 spin_lock_bh(&rb_queue->lock);
chetan lokef6fb8f102011-08-19 10:18:16 +0000531 pkc->delete_blk_timer = 1;
Veaceslav Falicoec6f8092013-11-29 09:53:23 +0100532 spin_unlock_bh(&rb_queue->lock);
chetan lokef6fb8f102011-08-19 10:18:16 +0000533
534 prb_del_retire_blk_timer(pkc);
535}
536
537static void prb_init_blk_timer(struct packet_sock *po,
chetan lokebc59ba32011-08-25 10:43:30 +0000538 struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f102011-08-19 10:18:16 +0000539 void (*func) (unsigned long))
540{
541 init_timer(&pkc->retire_blk_timer);
542 pkc->retire_blk_timer.data = (long)po;
543 pkc->retire_blk_timer.function = func;
544 pkc->retire_blk_timer.expires = jiffies;
545}
546
Maninder Singhe8e85cc2015-06-22 12:39:16 +0530547static void prb_setup_retire_blk_timer(struct packet_sock *po)
chetan lokef6fb8f102011-08-19 10:18:16 +0000548{
chetan lokebc59ba32011-08-25 10:43:30 +0000549 struct tpacket_kbdq_core *pkc;
chetan lokef6fb8f102011-08-19 10:18:16 +0000550
Maninder Singhe8e85cc2015-06-22 12:39:16 +0530551 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
chetan lokef6fb8f102011-08-19 10:18:16 +0000552 prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);
553}
554
555static int prb_calc_retire_blk_tmo(struct packet_sock *po,
556 int blk_size_in_bytes)
557{
558 struct net_device *dev;
559 unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
David Decotigny7cad1ba2016-02-24 10:58:10 -0800560 struct ethtool_link_ksettings ecmd;
Jiri Pirko4bc71cb2011-09-03 03:34:30 +0000561 int err;
chetan lokef6fb8f102011-08-19 10:18:16 +0000562
Jiri Pirko4bc71cb2011-09-03 03:34:30 +0000563 rtnl_lock();
564 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
565 if (unlikely(!dev)) {
566 rtnl_unlock();
chetan lokef6fb8f102011-08-19 10:18:16 +0000567 return DEFAULT_PRB_RETIRE_TOV;
Jiri Pirko4bc71cb2011-09-03 03:34:30 +0000568 }
David Decotigny7cad1ba2016-02-24 10:58:10 -0800569 err = __ethtool_get_link_ksettings(dev, &ecmd);
Jiri Pirko4bc71cb2011-09-03 03:34:30 +0000570 rtnl_unlock();
571 if (!err) {
Jiri Pirko4bc71cb2011-09-03 03:34:30 +0000572 /*
573 * If the link speed is so slow you don't really
574 * need to worry about perf anyways
575 */
David Decotigny7cad1ba2016-02-24 10:58:10 -0800576 if (ecmd.base.speed < SPEED_1000 ||
577 ecmd.base.speed == SPEED_UNKNOWN) {
Jiri Pirko4bc71cb2011-09-03 03:34:30 +0000578 return DEFAULT_PRB_RETIRE_TOV;
parav.pandit@emulex.come440cf22012-06-27 03:56:12 +0000579 } else {
580 msec = 1;
David Decotigny7cad1ba2016-02-24 10:58:10 -0800581 div = ecmd.base.speed / 1000;
chetan lokef6fb8f102011-08-19 10:18:16 +0000582 }
583 }
584
585 mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
586
587 if (div)
588 mbits /= div;
589
590 tmo = mbits * msec;
591
592 if (div)
593 return tmo+1;
594 return tmo;
595}
596
chetan lokebc59ba32011-08-25 10:43:30 +0000597static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
chetan lokef6fb8f102011-08-19 10:18:16 +0000598 union tpacket_req_u *req_u)
599{
600 p1->feature_req_word = req_u->req3.tp_feature_req_word;
601}
602
603static void init_prb_bdqc(struct packet_sock *po,
604 struct packet_ring_buffer *rb,
605 struct pgv *pg_vec,
Maninder Singhe8e85cc2015-06-22 12:39:16 +0530606 union tpacket_req_u *req_u)
chetan lokef6fb8f102011-08-19 10:18:16 +0000607{
Duan Jiong22781a52013-12-06 13:29:36 +0800608 struct tpacket_kbdq_core *p1 = GET_PBDQC_FROM_RB(rb);
chetan lokebc59ba32011-08-25 10:43:30 +0000609 struct tpacket_block_desc *pbd;
chetan lokef6fb8f102011-08-19 10:18:16 +0000610
611 memset(p1, 0x0, sizeof(*p1));
612
613 p1->knxt_seq_num = 1;
614 p1->pkbdq = pg_vec;
chetan lokebc59ba32011-08-25 10:43:30 +0000615 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
Joe Perchese3192692012-06-03 17:41:40 +0000616 p1->pkblk_start = pg_vec[0].buffer;
chetan lokef6fb8f102011-08-19 10:18:16 +0000617 p1->kblk_size = req_u->req3.tp_block_size;
618 p1->knum_blocks = req_u->req3.tp_block_nr;
619 p1->hdrlen = po->tp_hdrlen;
620 p1->version = po->tp_version;
621 p1->last_kactive_blk_num = 0;
Daniel Borkmannee80fbf2013-04-19 06:12:29 +0000622 po->stats.stats3.tp_freeze_q_cnt = 0;
chetan lokef6fb8f102011-08-19 10:18:16 +0000623 if (req_u->req3.tp_retire_blk_tov)
624 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
625 else
626 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
627 req_u->req3.tp_block_size);
628 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
629 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
630
Eric Dumazetdc808112014-08-15 09:16:04 -0700631 p1->max_frame_len = p1->kblk_size - BLK_PLUS_PRIV(p1->blk_sizeof_priv);
chetan lokef6fb8f102011-08-19 10:18:16 +0000632 prb_init_ft_ops(p1, req_u);
Maninder Singhe8e85cc2015-06-22 12:39:16 +0530633 prb_setup_retire_blk_timer(po);
chetan lokef6fb8f102011-08-19 10:18:16 +0000634 prb_open_block(p1, pbd);
635}
636
637/* Do NOT update the last_blk_num first.
638 * Assumes sk_buff_head lock is held.
639 */
chetan lokebc59ba32011-08-25 10:43:30 +0000640static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
chetan lokef6fb8f102011-08-19 10:18:16 +0000641{
642 mod_timer(&pkc->retire_blk_timer,
643 jiffies + pkc->tov_in_jiffies);
644 pkc->last_kactive_blk_num = pkc->kactive_blk_num;
645}
646
647/*
648 * Timer logic:
649 * 1) We refresh the timer only when we open a block.
650 * By doing this we don't waste cycles refreshing the timer
651 * on packet-by-packet basis.
652 *
653 * With a 1MB block-size, on a 1Gbps line, it will take
654 * i) ~8 ms to fill a block + ii) memcpy etc.
655 * In this cut we are not accounting for the memcpy time.
656 *
657 * So, if the user sets the 'tmo' to 10ms then the timer
658 * will never fire while the block is still getting filled
659 * (which is what we want). However, the user could choose
660 * to close a block early and that's fine.
661 *
662 * But when the timer does fire, we check whether or not to refresh it.
663 * Since the tmo granularity is in msecs, it is not too expensive
664 * to refresh the timer, lets say every '8' msecs.
665 * Either the user can set the 'tmo' or we can derive it based on
666 * a) line-speed and b) block-size.
667 * prb_calc_retire_blk_tmo() calculates the tmo.
668 *
669 */
670static void prb_retire_rx_blk_timer_expired(unsigned long data)
671{
672 struct packet_sock *po = (struct packet_sock *)data;
Duan Jiong22781a52013-12-06 13:29:36 +0800673 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
chetan lokef6fb8f102011-08-19 10:18:16 +0000674 unsigned int frozen;
chetan lokebc59ba32011-08-25 10:43:30 +0000675 struct tpacket_block_desc *pbd;
chetan lokef6fb8f102011-08-19 10:18:16 +0000676
677 spin_lock(&po->sk.sk_receive_queue.lock);
678
679 frozen = prb_queue_frozen(pkc);
680 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
681
682 if (unlikely(pkc->delete_blk_timer))
683 goto out;
684
685 /* We only need to plug the race when the block is partially filled.
686 * tpacket_rcv:
687 * lock(); increment BLOCK_NUM_PKTS; unlock()
688 * copy_bits() is in progress ...
689 * timer fires on other cpu:
690 * we can't retire the current block because copy_bits
691 * is in progress.
692 *
693 */
694 if (BLOCK_NUM_PKTS(pbd)) {
695 while (atomic_read(&pkc->blk_fill_in_prog)) {
696 /* Waiting for skb_copy_bits to finish... */
697 cpu_relax();
698 }
699 }
700
701 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
702 if (!frozen) {
Alexander Drozdov41a50d62015-02-24 08:18:28 +0300703 if (!BLOCK_NUM_PKTS(pbd)) {
704 /* An empty block. Just refresh the timer. */
705 goto refresh_timer;
706 }
chetan lokef6fb8f102011-08-19 10:18:16 +0000707 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
708 if (!prb_dispatch_next_block(pkc, po))
709 goto refresh_timer;
710 else
711 goto out;
712 } else {
713 /* Case 1. Queue was frozen because user-space was
714 * lagging behind.
715 */
716 if (prb_curr_blk_in_use(pkc, pbd)) {
717 /*
718 * Ok, user-space is still behind.
719 * So just refresh the timer.
720 */
721 goto refresh_timer;
722 } else {
723 /* Case 2. queue was frozen,user-space caught up,
724 * now the link went idle && the timer fired.
725 * We don't have a block to close.So we open this
726 * block and restart the timer.
727 * opening a block thaws the queue,restarts timer
728 * Thawing/timer-refresh is a side effect.
729 */
730 prb_open_block(pkc, pbd);
731 goto out;
732 }
733 }
734 }
735
736refresh_timer:
737 _prb_refresh_rx_retire_blk_timer(pkc);
738
739out:
740 spin_unlock(&po->sk.sk_receive_queue.lock);
741}
742
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000743static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
chetan lokebc59ba32011-08-25 10:43:30 +0000744 struct tpacket_block_desc *pbd1, __u32 status)
chetan lokef6fb8f102011-08-19 10:18:16 +0000745{
746 /* Flush everything minus the block header */
747
748#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
749 u8 *start, *end;
750
751 start = (u8 *)pbd1;
752
753 /* Skip the block header(we know header WILL fit in 4K) */
754 start += PAGE_SIZE;
755
756 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
757 for (; start < end; start += PAGE_SIZE)
758 flush_dcache_page(pgv_to_page(start));
759
760 smp_wmb();
761#endif
762
763 /* Now update the block status. */
764
765 BLOCK_STATUS(pbd1) = status;
766
767 /* Flush the block header */
768
769#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
770 start = (u8 *)pbd1;
771 flush_dcache_page(pgv_to_page(start));
772
773 smp_wmb();
774#endif
775}
776
777/*
778 * Side effect:
779 *
780 * 1) flush the block
781 * 2) Increment active_blk_num
782 *
783 * Note:We DONT refresh the timer on purpose.
784 * Because almost always the next block will be opened.
785 */
chetan lokebc59ba32011-08-25 10:43:30 +0000786static void prb_close_block(struct tpacket_kbdq_core *pkc1,
787 struct tpacket_block_desc *pbd1,
chetan lokef6fb8f102011-08-19 10:18:16 +0000788 struct packet_sock *po, unsigned int stat)
789{
790 __u32 status = TP_STATUS_USER | stat;
791
792 struct tpacket3_hdr *last_pkt;
chetan lokebc59ba32011-08-25 10:43:30 +0000793 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
Dan Collinsda413ee2014-12-19 16:49:25 +1300794 struct sock *sk = &po->sk;
chetan lokef6fb8f102011-08-19 10:18:16 +0000795
Daniel Borkmannee80fbf2013-04-19 06:12:29 +0000796 if (po->stats.stats3.tp_drops)
chetan lokef6fb8f102011-08-19 10:18:16 +0000797 status |= TP_STATUS_LOSING;
798
799 last_pkt = (struct tpacket3_hdr *)pkc1->prev;
800 last_pkt->tp_next_offset = 0;
801
802 /* Get the ts of the last pkt */
803 if (BLOCK_NUM_PKTS(pbd1)) {
804 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
805 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
806 } else {
Alexander Drozdov41a50d62015-02-24 08:18:28 +0300807 /* Ok, we tmo'd - so get the current time.
808 *
809 * It shouldn't really happen as we don't close empty
810 * blocks. See prb_retire_rx_blk_timer_expired().
811 */
chetan lokef6fb8f102011-08-19 10:18:16 +0000812 struct timespec ts;
813 getnstimeofday(&ts);
814 h1->ts_last_pkt.ts_sec = ts.tv_sec;
815 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
816 }
817
818 smp_wmb();
819
820 /* Flush the block */
821 prb_flush_block(pkc1, pbd1, status);
822
Dan Collinsda413ee2014-12-19 16:49:25 +1300823 sk->sk_data_ready(sk);
824
chetan lokef6fb8f102011-08-19 10:18:16 +0000825 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
826}
827
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000828static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
chetan lokef6fb8f102011-08-19 10:18:16 +0000829{
830 pkc->reset_pending_on_curr_blk = 0;
831}
832
833/*
834 * Side effect of opening a block:
835 *
836 * 1) prb_queue is thawed.
837 * 2) retire_blk_timer is refreshed.
838 *
839 */
chetan lokebc59ba32011-08-25 10:43:30 +0000840static void prb_open_block(struct tpacket_kbdq_core *pkc1,
841 struct tpacket_block_desc *pbd1)
chetan lokef6fb8f102011-08-19 10:18:16 +0000842{
843 struct timespec ts;
chetan lokebc59ba32011-08-25 10:43:30 +0000844 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
chetan lokef6fb8f102011-08-19 10:18:16 +0000845
846 smp_rmb();
847
Daniel Borkmann8da30562013-05-03 02:57:00 +0000848 /* We could have just memset this but we will lose the
849 * flexibility of making the priv area sticky
850 */
chetan lokef6fb8f102011-08-19 10:18:16 +0000851
Daniel Borkmann8da30562013-05-03 02:57:00 +0000852 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
853 BLOCK_NUM_PKTS(pbd1) = 0;
854 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
chetan lokef6fb8f102011-08-19 10:18:16 +0000855
Daniel Borkmann8da30562013-05-03 02:57:00 +0000856 getnstimeofday(&ts);
chetan lokef6fb8f102011-08-19 10:18:16 +0000857
Daniel Borkmann8da30562013-05-03 02:57:00 +0000858 h1->ts_first_pkt.ts_sec = ts.tv_sec;
859 h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
chetan lokef6fb8f102011-08-19 10:18:16 +0000860
Daniel Borkmann8da30562013-05-03 02:57:00 +0000861 pkc1->pkblk_start = (char *)pbd1;
862 pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
863
864 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
865 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
866
867 pbd1->version = pkc1->version;
868 pkc1->prev = pkc1->nxt_offset;
869 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
870
871 prb_thaw_queue(pkc1);
872 _prb_refresh_rx_retire_blk_timer(pkc1);
873
874 smp_wmb();
chetan lokef6fb8f102011-08-19 10:18:16 +0000875}
876
877/*
878 * Queue freeze logic:
879 * 1) Assume tp_block_nr = 8 blocks.
880 * 2) At time 't0', user opens Rx ring.
881 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
882 * 4) user-space is either sleeping or processing block '0'.
883 * 5) tpacket_rcv is currently filling block '7', since there is no space left,
884 * it will close block-7,loop around and try to fill block '0'.
885 * call-flow:
886 * __packet_lookup_frame_in_block
887 * prb_retire_current_block()
888 * prb_dispatch_next_block()
889 * |->(BLOCK_STATUS == USER) evaluates to true
890 * 5.1) Since block-0 is currently in-use, we just freeze the queue.
891 * 6) Now there are two cases:
892 * 6.1) Link goes idle right after the queue is frozen.
893 * But remember, the last open_block() refreshed the timer.
894 * When this timer expires,it will refresh itself so that we can
895 * re-open block-0 in near future.
896 * 6.2) Link is busy and keeps on receiving packets. This is a simple
897 * case and __packet_lookup_frame_in_block will check if block-0
898 * is free and can now be re-used.
899 */
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000900static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f102011-08-19 10:18:16 +0000901 struct packet_sock *po)
902{
903 pkc->reset_pending_on_curr_blk = 1;
Daniel Borkmannee80fbf2013-04-19 06:12:29 +0000904 po->stats.stats3.tp_freeze_q_cnt++;
chetan lokef6fb8f102011-08-19 10:18:16 +0000905}
906
907#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
908
909/*
910 * If the next block is free then we will dispatch it
911 * and return a good offset.
912 * Else, we will freeze the queue.
913 * So, caller must check the return value.
914 */
chetan lokebc59ba32011-08-25 10:43:30 +0000915static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f102011-08-19 10:18:16 +0000916 struct packet_sock *po)
917{
chetan lokebc59ba32011-08-25 10:43:30 +0000918 struct tpacket_block_desc *pbd;
chetan lokef6fb8f102011-08-19 10:18:16 +0000919
920 smp_rmb();
921
922 /* 1. Get current block num */
923 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
924
925 /* 2. If this block is currently in_use then freeze the queue */
926 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
927 prb_freeze_queue(pkc, po);
928 return NULL;
929 }
930
931 /*
932 * 3.
933 * open this block and return the offset where the first packet
934 * needs to get stored.
935 */
936 prb_open_block(pkc, pbd);
937 return (void *)pkc->nxt_offset;
938}
939
chetan lokebc59ba32011-08-25 10:43:30 +0000940static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f102011-08-19 10:18:16 +0000941 struct packet_sock *po, unsigned int status)
942{
chetan lokebc59ba32011-08-25 10:43:30 +0000943 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
chetan lokef6fb8f102011-08-19 10:18:16 +0000944
945 /* retire/close the current block */
946 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
947 /*
948 * Plug the case where copy_bits() is in progress on
949 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
950 * have space to copy the pkt in the current block and
951 * called prb_retire_current_block()
952 *
953 * We don't need to worry about the TMO case because
954 * the timer-handler already handled this case.
955 */
956 if (!(status & TP_STATUS_BLK_TMO)) {
957 while (atomic_read(&pkc->blk_fill_in_prog)) {
958 /* Waiting for skb_copy_bits to finish... */
959 cpu_relax();
960 }
961 }
962 prb_close_block(pkc, pbd, po, status);
963 return;
964 }
chetan lokef6fb8f102011-08-19 10:18:16 +0000965}
966
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000967static int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,
chetan lokebc59ba32011-08-25 10:43:30 +0000968 struct tpacket_block_desc *pbd)
chetan lokef6fb8f102011-08-19 10:18:16 +0000969{
970 return TP_STATUS_USER & BLOCK_STATUS(pbd);
971}
972
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000973static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
chetan lokef6fb8f102011-08-19 10:18:16 +0000974{
975 return pkc->reset_pending_on_curr_blk;
976}
977
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000978static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
chetan lokef6fb8f102011-08-19 10:18:16 +0000979{
chetan lokebc59ba32011-08-25 10:43:30 +0000980 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
chetan lokef6fb8f102011-08-19 10:18:16 +0000981 atomic_dec(&pkc->blk_fill_in_prog);
982}
983
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000984static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f102011-08-19 10:18:16 +0000985 struct tpacket3_hdr *ppd)
986{
Tom Herbert3958afa1b2013-12-15 22:12:06 -0800987 ppd->hv1.tp_rxhash = skb_get_hash(pkc->skb);
chetan lokef6fb8f102011-08-19 10:18:16 +0000988}
989
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000990static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f102011-08-19 10:18:16 +0000991 struct tpacket3_hdr *ppd)
992{
993 ppd->hv1.tp_rxhash = 0;
994}
995
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000996static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f102011-08-19 10:18:16 +0000997 struct tpacket3_hdr *ppd)
998{
Jiri Pirkodf8a39d2015-01-13 17:13:44 +0100999 if (skb_vlan_tag_present(pkc->skb)) {
1000 ppd->hv1.tp_vlan_tci = skb_vlan_tag_get(pkc->skb);
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +09001001 ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->vlan_proto);
1002 ppd->tp_status = TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
chetan lokef6fb8f102011-08-19 10:18:16 +00001003 } else {
danborkmann@iogearbox.net9e670302012-08-20 03:34:03 +00001004 ppd->hv1.tp_vlan_tci = 0;
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +09001005 ppd->hv1.tp_vlan_tpid = 0;
danborkmann@iogearbox.net9e670302012-08-20 03:34:03 +00001006 ppd->tp_status = TP_STATUS_AVAILABLE;
chetan lokef6fb8f102011-08-19 10:18:16 +00001007 }
1008}
1009
chetan lokebc59ba32011-08-25 10:43:30 +00001010static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f102011-08-19 10:18:16 +00001011 struct tpacket3_hdr *ppd)
1012{
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +09001013 ppd->hv1.tp_padding = 0;
chetan lokef6fb8f102011-08-19 10:18:16 +00001014 prb_fill_vlan_info(pkc, ppd);
1015
1016 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
1017 prb_fill_rxhash(pkc, ppd);
1018 else
1019 prb_clear_rxhash(pkc, ppd);
1020}
1021
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001022static void prb_fill_curr_block(char *curr,
chetan lokebc59ba32011-08-25 10:43:30 +00001023 struct tpacket_kbdq_core *pkc,
1024 struct tpacket_block_desc *pbd,
chetan lokef6fb8f102011-08-19 10:18:16 +00001025 unsigned int len)
1026{
1027 struct tpacket3_hdr *ppd;
1028
1029 ppd = (struct tpacket3_hdr *)curr;
1030 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
1031 pkc->prev = curr;
1032 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
1033 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
1034 BLOCK_NUM_PKTS(pbd) += 1;
1035 atomic_inc(&pkc->blk_fill_in_prog);
1036 prb_run_all_ft_ops(pkc, ppd);
1037}
1038
1039/* Assumes caller has the sk->rx_queue.lock */
1040static void *__packet_lookup_frame_in_block(struct packet_sock *po,
1041 struct sk_buff *skb,
1042 int status,
1043 unsigned int len
1044 )
1045{
chetan lokebc59ba32011-08-25 10:43:30 +00001046 struct tpacket_kbdq_core *pkc;
1047 struct tpacket_block_desc *pbd;
chetan lokef6fb8f102011-08-19 10:18:16 +00001048 char *curr, *end;
1049
Joe Perchese3192692012-06-03 17:41:40 +00001050 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
chetan lokef6fb8f102011-08-19 10:18:16 +00001051 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1052
1053 /* Queue is frozen when user space is lagging behind */
1054 if (prb_queue_frozen(pkc)) {
1055 /*
1056 * Check if that last block which caused the queue to freeze,
1057 * is still in_use by user-space.
1058 */
1059 if (prb_curr_blk_in_use(pkc, pbd)) {
1060 /* Can't record this packet */
1061 return NULL;
1062 } else {
1063 /*
1064 * Ok, the block was released by user-space.
1065 * Now let's open that block.
1066 * opening a block also thaws the queue.
1067 * Thawing is a side effect.
1068 */
1069 prb_open_block(pkc, pbd);
1070 }
1071 }
1072
1073 smp_mb();
1074 curr = pkc->nxt_offset;
1075 pkc->skb = skb;
Joe Perchese3192692012-06-03 17:41:40 +00001076 end = (char *)pbd + pkc->kblk_size;
chetan lokef6fb8f102011-08-19 10:18:16 +00001077
1078 /* first try the current block */
1079 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
1080 prb_fill_curr_block(curr, pkc, pbd, len);
1081 return (void *)curr;
1082 }
1083
1084 /* Ok, close the current block */
1085 prb_retire_current_block(pkc, po, 0);
1086
1087 /* Now, try to dispatch the next block */
1088 curr = (char *)prb_dispatch_next_block(pkc, po);
1089 if (curr) {
1090 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1091 prb_fill_curr_block(curr, pkc, pbd, len);
1092 return (void *)curr;
1093 }
1094
1095 /*
1096 * No free blocks are available.user_space hasn't caught up yet.
1097 * Queue was just frozen and now this packet will get dropped.
1098 */
1099 return NULL;
1100}
1101
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001102static void *packet_current_rx_frame(struct packet_sock *po,
chetan lokef6fb8f102011-08-19 10:18:16 +00001103 struct sk_buff *skb,
1104 int status, unsigned int len)
1105{
1106 char *curr = NULL;
1107 switch (po->tp_version) {
1108 case TPACKET_V1:
1109 case TPACKET_V2:
1110 curr = packet_lookup_frame(po, &po->rx_ring,
1111 po->rx_ring.head, status);
1112 return curr;
1113 case TPACKET_V3:
1114 return __packet_lookup_frame_in_block(po, skb, status, len);
1115 default:
1116 WARN(1, "TPACKET version not supported\n");
1117 BUG();
Ying Xue99aa3472012-08-06 16:27:10 +00001118 return NULL;
chetan lokef6fb8f102011-08-19 10:18:16 +00001119 }
1120}
1121
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001122static void *prb_lookup_block(struct packet_sock *po,
chetan lokef6fb8f102011-08-19 10:18:16 +00001123 struct packet_ring_buffer *rb,
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001124 unsigned int idx,
chetan lokef6fb8f102011-08-19 10:18:16 +00001125 int status)
1126{
chetan lokebc59ba32011-08-25 10:43:30 +00001127 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001128 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
chetan lokef6fb8f102011-08-19 10:18:16 +00001129
1130 if (status != BLOCK_STATUS(pbd))
1131 return NULL;
1132 return pbd;
1133}
1134
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001135static int prb_previous_blk_num(struct packet_ring_buffer *rb)
chetan lokef6fb8f102011-08-19 10:18:16 +00001136{
1137 unsigned int prev;
1138 if (rb->prb_bdqc.kactive_blk_num)
1139 prev = rb->prb_bdqc.kactive_blk_num-1;
1140 else
1141 prev = rb->prb_bdqc.knum_blocks-1;
1142 return prev;
1143}
1144
1145/* Assumes caller has held the rx_queue.lock */
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001146static void *__prb_previous_block(struct packet_sock *po,
chetan lokef6fb8f102011-08-19 10:18:16 +00001147 struct packet_ring_buffer *rb,
1148 int status)
1149{
1150 unsigned int previous = prb_previous_blk_num(rb);
1151 return prb_lookup_block(po, rb, previous, status);
1152}
1153
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001154static void *packet_previous_rx_frame(struct packet_sock *po,
chetan lokef6fb8f102011-08-19 10:18:16 +00001155 struct packet_ring_buffer *rb,
1156 int status)
1157{
1158 if (po->tp_version <= TPACKET_V2)
1159 return packet_previous_frame(po, rb, status);
1160
1161 return __prb_previous_block(po, rb, status);
1162}
1163
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001164static void packet_increment_rx_head(struct packet_sock *po,
chetan lokef6fb8f102011-08-19 10:18:16 +00001165 struct packet_ring_buffer *rb)
1166{
1167 switch (po->tp_version) {
1168 case TPACKET_V1:
1169 case TPACKET_V2:
1170 return packet_increment_head(rb);
1171 case TPACKET_V3:
1172 default:
1173 WARN(1, "TPACKET version not supported.\n");
1174 BUG();
1175 return;
1176 }
1177}
1178
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001179static void *packet_previous_frame(struct packet_sock *po,
Johann Baudy69e3c752009-05-18 22:11:22 -07001180 struct packet_ring_buffer *rb,
1181 int status)
1182{
1183 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1184 return packet_lookup_frame(po, rb, previous, status);
1185}
1186
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001187static void packet_increment_head(struct packet_ring_buffer *buff)
Johann Baudy69e3c752009-05-18 22:11:22 -07001188{
1189 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1190}
1191
Daniel Borkmannb0138402014-01-15 16:25:36 +01001192static void packet_inc_pending(struct packet_ring_buffer *rb)
1193{
1194 this_cpu_inc(*rb->pending_refcnt);
1195}
1196
1197static void packet_dec_pending(struct packet_ring_buffer *rb)
1198{
1199 this_cpu_dec(*rb->pending_refcnt);
1200}
1201
1202static unsigned int packet_read_pending(const struct packet_ring_buffer *rb)
1203{
1204 unsigned int refcnt = 0;
1205 int cpu;
1206
1207 /* We don't use pending refcount in rx_ring. */
1208 if (rb->pending_refcnt == NULL)
1209 return 0;
1210
1211 for_each_possible_cpu(cpu)
1212 refcnt += *per_cpu_ptr(rb->pending_refcnt, cpu);
1213
1214 return refcnt;
1215}
1216
1217static int packet_alloc_pending(struct packet_sock *po)
1218{
1219 po->rx_ring.pending_refcnt = NULL;
1220
1221 po->tx_ring.pending_refcnt = alloc_percpu(unsigned int);
1222 if (unlikely(po->tx_ring.pending_refcnt == NULL))
1223 return -ENOBUFS;
1224
1225 return 0;
1226}
1227
1228static void packet_free_pending(struct packet_sock *po)
1229{
1230 free_percpu(po->tx_ring.pending_refcnt);
1231}
1232
Willem de Bruijn99547292015-05-12 11:56:47 -04001233#define ROOM_POW_OFF 2
1234#define ROOM_NONE 0x0
1235#define ROOM_LOW 0x1
1236#define ROOM_NORMAL 0x2
1237
1238static bool __tpacket_has_room(struct packet_sock *po, int pow_off)
1239{
1240 int idx, len;
1241
1242 len = po->rx_ring.frame_max + 1;
1243 idx = po->rx_ring.head;
1244 if (pow_off)
1245 idx += len >> pow_off;
1246 if (idx >= len)
1247 idx -= len;
1248 return packet_lookup_frame(po, &po->rx_ring, idx, TP_STATUS_KERNEL);
1249}
1250
1251static bool __tpacket_v3_has_room(struct packet_sock *po, int pow_off)
1252{
1253 int idx, len;
1254
1255 len = po->rx_ring.prb_bdqc.knum_blocks;
1256 idx = po->rx_ring.prb_bdqc.kactive_blk_num;
1257 if (pow_off)
1258 idx += len >> pow_off;
1259 if (idx >= len)
1260 idx -= len;
1261 return prb_lookup_block(po, &po->rx_ring, idx, TP_STATUS_KERNEL);
1262}
1263
Willem de Bruijn2ccdbaa2015-05-12 11:56:48 -04001264static int __packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001265{
1266 struct sock *sk = &po->sk;
Willem de Bruijn99547292015-05-12 11:56:47 -04001267 int ret = ROOM_NONE;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001268
Willem de Bruijn99547292015-05-12 11:56:47 -04001269 if (po->prot_hook.func != tpacket_rcv) {
1270 int avail = sk->sk_rcvbuf - atomic_read(&sk->sk_rmem_alloc)
Willem de Bruijn2ccdbaa2015-05-12 11:56:48 -04001271 - (skb ? skb->truesize : 0);
Willem de Bruijn99547292015-05-12 11:56:47 -04001272 if (avail > (sk->sk_rcvbuf >> ROOM_POW_OFF))
1273 return ROOM_NORMAL;
1274 else if (avail > 0)
1275 return ROOM_LOW;
1276 else
1277 return ROOM_NONE;
1278 }
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001279
Willem de Bruijn99547292015-05-12 11:56:47 -04001280 if (po->tp_version == TPACKET_V3) {
1281 if (__tpacket_v3_has_room(po, ROOM_POW_OFF))
1282 ret = ROOM_NORMAL;
1283 else if (__tpacket_v3_has_room(po, 0))
1284 ret = ROOM_LOW;
1285 } else {
1286 if (__tpacket_has_room(po, ROOM_POW_OFF))
1287 ret = ROOM_NORMAL;
1288 else if (__tpacket_has_room(po, 0))
1289 ret = ROOM_LOW;
1290 }
Willem de Bruijn2ccdbaa2015-05-12 11:56:48 -04001291
1292 return ret;
1293}
1294
1295static int packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1296{
1297 int ret;
1298 bool has_room;
1299
Willem de Bruijn54d7c012015-05-14 15:25:02 -04001300 spin_lock_bh(&po->sk.sk_receive_queue.lock);
1301 ret = __packet_rcv_has_room(po, skb);
Willem de Bruijn2ccdbaa2015-05-12 11:56:48 -04001302 has_room = ret == ROOM_NORMAL;
1303 if (po->pressure == has_room)
Willem de Bruijn54d7c012015-05-14 15:25:02 -04001304 po->pressure = !has_room;
1305 spin_unlock_bh(&po->sk.sk_receive_queue.lock);
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001306
Willem de Bruijn99547292015-05-12 11:56:47 -04001307 return ret;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001308}
1309
Linus Torvalds1da177e2005-04-16 15:20:36 -07001310static void packet_sock_destruct(struct sock *sk)
1311{
Richard Cochraned85b562010-04-07 22:41:28 +00001312 skb_queue_purge(&sk->sk_error_queue);
1313
Ilpo Järvinen547b7922008-07-25 21:43:18 -07001314 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1315 WARN_ON(atomic_read(&sk->sk_wmem_alloc));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001316
1317 if (!sock_flag(sk, SOCK_DEAD)) {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001318 pr_err("Attempt to release alive packet socket: %p\n", sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001319 return;
1320 }
1321
Pavel Emelyanov17ab56a2007-11-10 21:38:48 -08001322 sk_refcnt_debug_dec(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001323}
1324
Willem de Bruijn3b3a5b02015-05-12 11:56:49 -04001325static bool fanout_flow_is_huge(struct packet_sock *po, struct sk_buff *skb)
1326{
1327 u32 rxhash;
1328 int i, count = 0;
1329
1330 rxhash = skb_get_hash(skb);
1331 for (i = 0; i < ROLLOVER_HLEN; i++)
1332 if (po->rollover->history[i] == rxhash)
1333 count++;
1334
1335 po->rollover->history[prandom_u32() % ROLLOVER_HLEN] = rxhash;
1336 return count > (ROLLOVER_HLEN >> 1);
1337}
1338
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001339static unsigned int fanout_demux_hash(struct packet_fanout *f,
1340 struct sk_buff *skb,
1341 unsigned int num)
David S. Millerdc99f602011-07-05 01:45:05 -07001342{
David S. Millereb70db82016-07-01 16:07:50 -04001343 return reciprocal_scale(__skb_get_hash_symmetric(skb), num);
David S. Millerdc99f602011-07-05 01:45:05 -07001344}
1345
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001346static unsigned int fanout_demux_lb(struct packet_fanout *f,
1347 struct sk_buff *skb,
1348 unsigned int num)
David S. Millerdc99f602011-07-05 01:45:05 -07001349{
Willem de Bruijn468479e2015-06-17 15:59:34 -04001350 unsigned int val = atomic_inc_return(&f->rr_cur);
David S. Millerdc99f602011-07-05 01:45:05 -07001351
Willem de Bruijn468479e2015-06-17 15:59:34 -04001352 return val % num;
David S. Millerdc99f602011-07-05 01:45:05 -07001353}
1354
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001355static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1356 struct sk_buff *skb,
1357 unsigned int num)
David S. Miller95ec3eb2011-07-06 01:56:38 -07001358{
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001359 return smp_processor_id() % num;
1360}
David S. Miller95ec3eb2011-07-06 01:56:38 -07001361
Daniel Borkmann5df0ddf2013-08-28 22:13:09 +02001362static unsigned int fanout_demux_rnd(struct packet_fanout *f,
1363 struct sk_buff *skb,
1364 unsigned int num)
1365{
Daniel Borkmannf337db62014-01-22 02:29:39 +01001366 return prandom_u32_max(num);
Daniel Borkmann5df0ddf2013-08-28 22:13:09 +02001367}
1368
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001369static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1370 struct sk_buff *skb,
Willem de Bruijnad377ca2015-05-12 11:56:45 -04001371 unsigned int idx, bool try_self,
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001372 unsigned int num)
1373{
Willem de Bruijn4633c9e2015-05-17 19:44:02 -04001374 struct packet_sock *po, *po_next, *po_skip = NULL;
Willem de Bruijna9b63912015-05-12 11:56:50 -04001375 unsigned int i, j, room = ROOM_NONE;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001376
Willem de Bruijn0648ab72015-05-12 11:56:46 -04001377 po = pkt_sk(f->arr[idx]);
Willem de Bruijn3b3a5b02015-05-12 11:56:49 -04001378
1379 if (try_self) {
1380 room = packet_rcv_has_room(po, skb);
1381 if (room == ROOM_NORMAL ||
1382 (room == ROOM_LOW && !fanout_flow_is_huge(po, skb)))
1383 return idx;
Willem de Bruijn4633c9e2015-05-17 19:44:02 -04001384 po_skip = po;
Willem de Bruijn3b3a5b02015-05-12 11:56:49 -04001385 }
Willem de Bruijnad377ca2015-05-12 11:56:45 -04001386
Willem de Bruijn0648ab72015-05-12 11:56:46 -04001387 i = j = min_t(int, po->rollover->sock, num - 1);
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001388 do {
Willem de Bruijn2ccdbaa2015-05-12 11:56:48 -04001389 po_next = pkt_sk(f->arr[i]);
Willem de Bruijn4633c9e2015-05-17 19:44:02 -04001390 if (po_next != po_skip && !po_next->pressure &&
Willem de Bruijn2ccdbaa2015-05-12 11:56:48 -04001391 packet_rcv_has_room(po_next, skb) == ROOM_NORMAL) {
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001392 if (i != j)
Willem de Bruijn0648ab72015-05-12 11:56:46 -04001393 po->rollover->sock = i;
Willem de Bruijna9b63912015-05-12 11:56:50 -04001394 atomic_long_inc(&po->rollover->num);
1395 if (room == ROOM_LOW)
1396 atomic_long_inc(&po->rollover->num_huge);
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001397 return i;
1398 }
Willem de Bruijnad377ca2015-05-12 11:56:45 -04001399
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001400 if (++i == num)
1401 i = 0;
1402 } while (i != j);
1403
Willem de Bruijna9b63912015-05-12 11:56:50 -04001404 atomic_long_inc(&po->rollover->num_failed);
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001405 return idx;
1406}
1407
Neil Horman2d360972014-01-22 16:01:44 -05001408static unsigned int fanout_demux_qm(struct packet_fanout *f,
1409 struct sk_buff *skb,
1410 unsigned int num)
1411{
1412 return skb_get_queue_mapping(skb) % num;
1413}
1414
Willem de Bruijn47dceb82015-08-14 22:31:34 -04001415static unsigned int fanout_demux_bpf(struct packet_fanout *f,
1416 struct sk_buff *skb,
1417 unsigned int num)
1418{
1419 struct bpf_prog *prog;
1420 unsigned int ret = 0;
1421
1422 rcu_read_lock();
1423 prog = rcu_dereference(f->bpf_prog);
1424 if (prog)
Alexei Starovoitovff936a02015-10-07 10:55:41 -07001425 ret = bpf_prog_run_clear_cb(prog, skb) % num;
Willem de Bruijn47dceb82015-08-14 22:31:34 -04001426 rcu_read_unlock();
1427
1428 return ret;
1429}
1430
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001431static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1432{
1433 return f->flags & (flag >> 8);
David S. Miller95ec3eb2011-07-06 01:56:38 -07001434}
1435
David S. Miller95ec3eb2011-07-06 01:56:38 -07001436static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1437 struct packet_type *pt, struct net_device *orig_dev)
David S. Millerdc99f602011-07-05 01:45:05 -07001438{
1439 struct packet_fanout *f = pt->af_packet_priv;
Eric Dumazetf98f4512015-06-16 07:59:11 -07001440 unsigned int num = READ_ONCE(f->num_members);
Eric W. Biederman19bcf9f2015-10-09 13:44:54 -05001441 struct net *net = read_pnet(&f->net);
David S. Millerdc99f602011-07-05 01:45:05 -07001442 struct packet_sock *po;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001443 unsigned int idx;
David S. Millerdc99f602011-07-05 01:45:05 -07001444
Eric W. Biederman19bcf9f2015-10-09 13:44:54 -05001445 if (!net_eq(dev_net(dev), net) || !num) {
David S. Millerdc99f602011-07-05 01:45:05 -07001446 kfree_skb(skb);
1447 return 0;
1448 }
1449
Alexander Drozdov3f34b242015-02-20 08:24:27 +03001450 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
Eric W. Biederman19bcf9f2015-10-09 13:44:54 -05001451 skb = ip_check_defrag(net, skb, IP_DEFRAG_AF_PACKET);
Alexander Drozdov3f34b242015-02-20 08:24:27 +03001452 if (!skb)
1453 return 0;
1454 }
David S. Miller95ec3eb2011-07-06 01:56:38 -07001455 switch (f->type) {
1456 case PACKET_FANOUT_HASH:
1457 default:
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001458 idx = fanout_demux_hash(f, skb, num);
David S. Miller95ec3eb2011-07-06 01:56:38 -07001459 break;
1460 case PACKET_FANOUT_LB:
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001461 idx = fanout_demux_lb(f, skb, num);
David S. Miller95ec3eb2011-07-06 01:56:38 -07001462 break;
1463 case PACKET_FANOUT_CPU:
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001464 idx = fanout_demux_cpu(f, skb, num);
1465 break;
Daniel Borkmann5df0ddf2013-08-28 22:13:09 +02001466 case PACKET_FANOUT_RND:
1467 idx = fanout_demux_rnd(f, skb, num);
1468 break;
Neil Horman2d360972014-01-22 16:01:44 -05001469 case PACKET_FANOUT_QM:
1470 idx = fanout_demux_qm(f, skb, num);
1471 break;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001472 case PACKET_FANOUT_ROLLOVER:
Willem de Bruijnad377ca2015-05-12 11:56:45 -04001473 idx = fanout_demux_rollover(f, skb, 0, false, num);
David S. Miller95ec3eb2011-07-06 01:56:38 -07001474 break;
Willem de Bruijn47dceb82015-08-14 22:31:34 -04001475 case PACKET_FANOUT_CBPF:
Willem de Bruijnf2e52092015-08-14 22:31:35 -04001476 case PACKET_FANOUT_EBPF:
Willem de Bruijn47dceb82015-08-14 22:31:34 -04001477 idx = fanout_demux_bpf(f, skb, num);
1478 break;
David S. Miller7736d332011-07-05 01:43:20 -07001479 }
1480
Willem de Bruijnad377ca2015-05-12 11:56:45 -04001481 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER))
1482 idx = fanout_demux_rollover(f, skb, idx, true, num);
David S. Millerdc99f602011-07-05 01:45:05 -07001483
Willem de Bruijnad377ca2015-05-12 11:56:45 -04001484 po = pkt_sk(f->arr[idx]);
David S. Millerdc99f602011-07-05 01:45:05 -07001485 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1486}
1487
Pavel Emelyanovfff33212012-08-16 05:36:48 +00001488DEFINE_MUTEX(fanout_mutex);
1489EXPORT_SYMBOL_GPL(fanout_mutex);
David S. Millerdc99f602011-07-05 01:45:05 -07001490static LIST_HEAD(fanout_list);
1491
1492static void __fanout_link(struct sock *sk, struct packet_sock *po)
1493{
1494 struct packet_fanout *f = po->fanout;
1495
1496 spin_lock(&f->lock);
1497 f->arr[f->num_members] = sk;
1498 smp_wmb();
1499 f->num_members++;
Anoob Soman9206efc2017-02-15 20:25:39 +00001500 if (f->num_members == 1)
1501 dev_add_pack(&f->prot_hook);
David S. Millerdc99f602011-07-05 01:45:05 -07001502 spin_unlock(&f->lock);
1503}
1504
1505static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1506{
1507 struct packet_fanout *f = po->fanout;
1508 int i;
1509
1510 spin_lock(&f->lock);
1511 for (i = 0; i < f->num_members; i++) {
1512 if (f->arr[i] == sk)
1513 break;
1514 }
1515 BUG_ON(i >= f->num_members);
1516 f->arr[i] = f->arr[f->num_members - 1];
1517 f->num_members--;
Anoob Soman9206efc2017-02-15 20:25:39 +00001518 if (f->num_members == 0)
1519 __dev_remove_pack(&f->prot_hook);
David S. Millerdc99f602011-07-05 01:45:05 -07001520 spin_unlock(&f->lock);
1521}
1522
Weilong Chend4dd8ae2013-12-23 11:31:38 +08001523static bool match_fanout_group(struct packet_type *ptype, struct sock *sk)
Eric Leblondc0de08d2012-08-16 22:02:58 +00001524{
Eric Dumazet161642e2015-10-09 11:29:32 -07001525 if (sk->sk_family != PF_PACKET)
1526 return false;
Eric Leblondc0de08d2012-08-16 22:02:58 +00001527
Eric Dumazet161642e2015-10-09 11:29:32 -07001528 return ptype->af_packet_priv == pkt_sk(sk)->fanout;
Eric Leblondc0de08d2012-08-16 22:02:58 +00001529}
1530
Willem de Bruijn47dceb82015-08-14 22:31:34 -04001531static void fanout_init_data(struct packet_fanout *f)
1532{
1533 switch (f->type) {
1534 case PACKET_FANOUT_LB:
1535 atomic_set(&f->rr_cur, 0);
1536 break;
1537 case PACKET_FANOUT_CBPF:
Willem de Bruijnf2e52092015-08-14 22:31:35 -04001538 case PACKET_FANOUT_EBPF:
Willem de Bruijn47dceb82015-08-14 22:31:34 -04001539 RCU_INIT_POINTER(f->bpf_prog, NULL);
1540 break;
1541 }
1542}
1543
1544static void __fanout_set_data_bpf(struct packet_fanout *f, struct bpf_prog *new)
1545{
1546 struct bpf_prog *old;
1547
1548 spin_lock(&f->lock);
1549 old = rcu_dereference_protected(f->bpf_prog, lockdep_is_held(&f->lock));
1550 rcu_assign_pointer(f->bpf_prog, new);
1551 spin_unlock(&f->lock);
1552
1553 if (old) {
1554 synchronize_net();
1555 bpf_prog_destroy(old);
1556 }
1557}
1558
1559static int fanout_set_data_cbpf(struct packet_sock *po, char __user *data,
1560 unsigned int len)
1561{
1562 struct bpf_prog *new;
1563 struct sock_fprog fprog;
1564 int ret;
1565
1566 if (sock_flag(&po->sk, SOCK_FILTER_LOCKED))
1567 return -EPERM;
1568 if (len != sizeof(fprog))
1569 return -EINVAL;
1570 if (copy_from_user(&fprog, data, len))
1571 return -EFAULT;
1572
Daniel Borkmannbab18992015-10-02 15:17:33 +02001573 ret = bpf_prog_create_from_user(&new, &fprog, NULL, false);
Willem de Bruijn47dceb82015-08-14 22:31:34 -04001574 if (ret)
1575 return ret;
1576
1577 __fanout_set_data_bpf(po->fanout, new);
1578 return 0;
1579}
1580
Willem de Bruijnf2e52092015-08-14 22:31:35 -04001581static int fanout_set_data_ebpf(struct packet_sock *po, char __user *data,
1582 unsigned int len)
1583{
1584 struct bpf_prog *new;
1585 u32 fd;
1586
1587 if (sock_flag(&po->sk, SOCK_FILTER_LOCKED))
1588 return -EPERM;
1589 if (len != sizeof(fd))
1590 return -EINVAL;
1591 if (copy_from_user(&fd, data, len))
1592 return -EFAULT;
1593
Daniel Borkmann113214b2016-06-30 17:24:44 +02001594 new = bpf_prog_get_type(fd, BPF_PROG_TYPE_SOCKET_FILTER);
Willem de Bruijnf2e52092015-08-14 22:31:35 -04001595 if (IS_ERR(new))
1596 return PTR_ERR(new);
Willem de Bruijnf2e52092015-08-14 22:31:35 -04001597
1598 __fanout_set_data_bpf(po->fanout, new);
1599 return 0;
1600}
1601
Willem de Bruijn47dceb82015-08-14 22:31:34 -04001602static int fanout_set_data(struct packet_sock *po, char __user *data,
1603 unsigned int len)
1604{
1605 switch (po->fanout->type) {
1606 case PACKET_FANOUT_CBPF:
1607 return fanout_set_data_cbpf(po, data, len);
Willem de Bruijnf2e52092015-08-14 22:31:35 -04001608 case PACKET_FANOUT_EBPF:
1609 return fanout_set_data_ebpf(po, data, len);
Willem de Bruijn47dceb82015-08-14 22:31:34 -04001610 default:
1611 return -EINVAL;
1612 };
1613}
1614
1615static void fanout_release_data(struct packet_fanout *f)
1616{
1617 switch (f->type) {
1618 case PACKET_FANOUT_CBPF:
Willem de Bruijnf2e52092015-08-14 22:31:35 -04001619 case PACKET_FANOUT_EBPF:
Willem de Bruijn47dceb82015-08-14 22:31:34 -04001620 __fanout_set_data_bpf(f, NULL);
1621 };
1622}
1623
David S. Miller7736d332011-07-05 01:43:20 -07001624static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
David S. Millerdc99f602011-07-05 01:45:05 -07001625{
Eric Dumazet722737f2017-02-14 09:03:51 -08001626 struct packet_rollover *rollover = NULL;
David S. Millerdc99f602011-07-05 01:45:05 -07001627 struct packet_sock *po = pkt_sk(sk);
1628 struct packet_fanout *f, *match;
David S. Miller7736d332011-07-05 01:43:20 -07001629 u8 type = type_flags & 0xff;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001630 u8 flags = type_flags >> 8;
David S. Millerdc99f602011-07-05 01:45:05 -07001631 int err;
1632
1633 switch (type) {
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001634 case PACKET_FANOUT_ROLLOVER:
1635 if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1636 return -EINVAL;
David S. Millerdc99f602011-07-05 01:45:05 -07001637 case PACKET_FANOUT_HASH:
1638 case PACKET_FANOUT_LB:
David S. Miller95ec3eb2011-07-06 01:56:38 -07001639 case PACKET_FANOUT_CPU:
Daniel Borkmann5df0ddf2013-08-28 22:13:09 +02001640 case PACKET_FANOUT_RND:
Neil Horman2d360972014-01-22 16:01:44 -05001641 case PACKET_FANOUT_QM:
Willem de Bruijn47dceb82015-08-14 22:31:34 -04001642 case PACKET_FANOUT_CBPF:
Willem de Bruijnf2e52092015-08-14 22:31:35 -04001643 case PACKET_FANOUT_EBPF:
David S. Millerdc99f602011-07-05 01:45:05 -07001644 break;
1645 default:
1646 return -EINVAL;
1647 }
1648
Eric Dumazet722737f2017-02-14 09:03:51 -08001649 mutex_lock(&fanout_mutex);
David S. Millerdc99f602011-07-05 01:45:05 -07001650
Eric Dumazet722737f2017-02-14 09:03:51 -08001651 err = -EINVAL;
1652 if (!po->running)
1653 goto out;
1654
1655 err = -EALREADY;
David S. Millerdc99f602011-07-05 01:45:05 -07001656 if (po->fanout)
Eric Dumazet722737f2017-02-14 09:03:51 -08001657 goto out;
David S. Millerdc99f602011-07-05 01:45:05 -07001658
Willem de Bruijn4633c9e2015-05-17 19:44:02 -04001659 if (type == PACKET_FANOUT_ROLLOVER ||
1660 (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)) {
Eric Dumazet722737f2017-02-14 09:03:51 -08001661 err = -ENOMEM;
1662 rollover = kzalloc(sizeof(*rollover), GFP_KERNEL);
1663 if (!rollover)
1664 goto out;
1665 atomic_long_set(&rollover->num, 0);
1666 atomic_long_set(&rollover->num_huge, 0);
1667 atomic_long_set(&rollover->num_failed, 0);
1668 po->rollover = rollover;
Willem de Bruijn0648ab72015-05-12 11:56:46 -04001669 }
1670
David S. Millerdc99f602011-07-05 01:45:05 -07001671 match = NULL;
1672 list_for_each_entry(f, &fanout_list, list) {
1673 if (f->id == id &&
1674 read_pnet(&f->net) == sock_net(sk)) {
1675 match = f;
1676 break;
1677 }
1678 }
Eric Dumazetafe62c62011-07-07 06:41:29 -07001679 err = -EINVAL;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001680 if (match && match->flags != flags)
Eric Dumazetafe62c62011-07-07 06:41:29 -07001681 goto out;
David S. Millerdc99f602011-07-05 01:45:05 -07001682 if (!match) {
Eric Dumazetafe62c62011-07-07 06:41:29 -07001683 err = -ENOMEM;
David S. Millerdc99f602011-07-05 01:45:05 -07001684 match = kzalloc(sizeof(*match), GFP_KERNEL);
Eric Dumazetafe62c62011-07-07 06:41:29 -07001685 if (!match)
1686 goto out;
1687 write_pnet(&match->net, sock_net(sk));
1688 match->id = id;
1689 match->type = type;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001690 match->flags = flags;
Eric Dumazetafe62c62011-07-07 06:41:29 -07001691 INIT_LIST_HEAD(&match->list);
1692 spin_lock_init(&match->lock);
1693 atomic_set(&match->sk_ref, 0);
Willem de Bruijn47dceb82015-08-14 22:31:34 -04001694 fanout_init_data(match);
Eric Dumazetafe62c62011-07-07 06:41:29 -07001695 match->prot_hook.type = po->prot_hook.type;
1696 match->prot_hook.dev = po->prot_hook.dev;
1697 match->prot_hook.func = packet_rcv_fanout;
1698 match->prot_hook.af_packet_priv = match;
Eric Leblondc0de08d2012-08-16 22:02:58 +00001699 match->prot_hook.id_match = match_fanout_group;
Eric Dumazetafe62c62011-07-07 06:41:29 -07001700 list_add(&match->list, &fanout_list);
1701 }
1702 err = -EINVAL;
1703 if (match->type == type &&
1704 match->prot_hook.type == po->prot_hook.type &&
1705 match->prot_hook.dev == po->prot_hook.dev) {
1706 err = -ENOSPC;
1707 if (atomic_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
1708 __dev_remove_pack(&po->prot_hook);
1709 po->fanout = match;
1710 atomic_inc(&match->sk_ref);
1711 __fanout_link(sk, po);
1712 err = 0;
David S. Millerdc99f602011-07-05 01:45:05 -07001713 }
1714 }
Eric Dumazetafe62c62011-07-07 06:41:29 -07001715out:
Eric Dumazet722737f2017-02-14 09:03:51 -08001716 if (err && rollover) {
1717 kfree(rollover);
Willem de Bruijn0648ab72015-05-12 11:56:46 -04001718 po->rollover = NULL;
1719 }
Eric Dumazet722737f2017-02-14 09:03:51 -08001720 mutex_unlock(&fanout_mutex);
David S. Millerdc99f602011-07-05 01:45:05 -07001721 return err;
1722}
1723
Anoob Soman9206efc2017-02-15 20:25:39 +00001724/* If pkt_sk(sk)->fanout->sk_ref is zero, this function removes
1725 * pkt_sk(sk)->fanout from fanout_list and returns pkt_sk(sk)->fanout.
1726 * It is the responsibility of the caller to call fanout_release_data() and
1727 * free the returned packet_fanout (after synchronize_net())
1728 */
1729static struct packet_fanout *fanout_release(struct sock *sk)
David S. Millerdc99f602011-07-05 01:45:05 -07001730{
1731 struct packet_sock *po = pkt_sk(sk);
1732 struct packet_fanout *f;
1733
Pavel Emelyanovfff33212012-08-16 05:36:48 +00001734 mutex_lock(&fanout_mutex);
Eric Dumazet722737f2017-02-14 09:03:51 -08001735 f = po->fanout;
1736 if (f) {
1737 po->fanout = NULL;
David S. Millerdc99f602011-07-05 01:45:05 -07001738
Anoob Soman9206efc2017-02-15 20:25:39 +00001739 if (atomic_dec_and_test(&f->sk_ref))
Eric Dumazet722737f2017-02-14 09:03:51 -08001740 list_del(&f->list);
Anoob Soman9206efc2017-02-15 20:25:39 +00001741 else
1742 f = NULL;
Eric Dumazet722737f2017-02-14 09:03:51 -08001743
1744 if (po->rollover)
1745 kfree_rcu(po->rollover, rcu);
David S. Millerdc99f602011-07-05 01:45:05 -07001746 }
1747 mutex_unlock(&fanout_mutex);
Anoob Soman9206efc2017-02-15 20:25:39 +00001748
1749 return f;
David S. Millerdc99f602011-07-05 01:45:05 -07001750}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001751
Daniel Borkmann3c70c132015-11-11 23:25:42 +01001752static bool packet_extra_vlan_len_allowed(const struct net_device *dev,
1753 struct sk_buff *skb)
1754{
1755 /* Earlier code assumed this would be a VLAN pkt, double-check
1756 * this now that we have the actual packet in hand. We can only
1757 * do this check on Ethernet devices.
1758 */
1759 if (unlikely(dev->type != ARPHRD_ETHER))
1760 return false;
1761
1762 skb_reset_mac_header(skb);
1763 return likely(eth_hdr(skb)->h_proto == htons(ETH_P_8021Q));
1764}
1765
Eric Dumazet90ddc4f2005-12-22 12:49:22 -08001766static const struct proto_ops packet_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001767
Eric Dumazet90ddc4f2005-12-22 12:49:22 -08001768static const struct proto_ops packet_ops_spkt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001769
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001770static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1771 struct packet_type *pt, struct net_device *orig_dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001772{
1773 struct sock *sk;
1774 struct sockaddr_pkt *spkt;
1775
1776 /*
1777 * When we registered the protocol we saved the socket in the data
1778 * field for just this event.
1779 */
1780
1781 sk = pt->af_packet_priv;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001782
Linus Torvalds1da177e2005-04-16 15:20:36 -07001783 /*
1784 * Yank back the headers [hope the device set this
1785 * right or kerboom...]
1786 *
1787 * Incoming packets have ll header pulled,
1788 * push it back.
1789 *
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -07001790 * For outgoing ones skb->data == skb_mac_header(skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001791 * so that this procedure is noop.
1792 */
1793
1794 if (skb->pkt_type == PACKET_LOOPBACK)
1795 goto out;
1796
Octavian Purdila09ad9bc2009-11-25 15:14:13 -08001797 if (!net_eq(dev_net(dev), sock_net(sk)))
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08001798 goto out;
1799
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001800 skb = skb_share_check(skb, GFP_ATOMIC);
1801 if (skb == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001802 goto oom;
1803
1804 /* drop any routing info */
Eric Dumazetadf30902009-06-02 05:19:30 +00001805 skb_dst_drop(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001806
Phil Oester84531c22005-07-12 11:57:52 -07001807 /* drop conntrack reference */
1808 nf_reset(skb);
1809
Herbert Xuffbc6112007-02-04 23:33:10 -08001810 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001811
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -07001812 skb_push(skb, skb->data - skb_mac_header(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001813
1814 /*
1815 * The SOCK_PACKET socket receives _all_ frames.
1816 */
1817
1818 spkt->spkt_family = dev->type;
1819 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1820 spkt->spkt_protocol = skb->protocol;
1821
1822 /*
1823 * Charge the memory to the socket. This is done specifically
1824 * to prevent sockets using all the memory up.
1825 */
1826
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001827 if (sock_queue_rcv_skb(sk, skb) == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001828 return 0;
1829
1830out:
1831 kfree_skb(skb);
1832oom:
1833 return 0;
1834}
1835
1836
1837/*
1838 * Output a raw packet to a device layer. This bypasses all the other
1839 * protocol layers and you must therefore supply it with a complete frame
1840 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001841
Ying Xue1b784142015-03-02 15:37:48 +08001842static int packet_sendmsg_spkt(struct socket *sock, struct msghdr *msg,
1843 size_t len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001844{
1845 struct sock *sk = sock->sk;
Steffen Hurrle342dfc32014-01-17 22:53:15 +01001846 DECLARE_SOCKADDR(struct sockaddr_pkt *, saddr, msg->msg_name);
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001847 struct sk_buff *skb = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001848 struct net_device *dev;
Soheil Hassas Yeganehc14ac942016-04-02 23:08:12 -04001849 struct sockcm_cookie sockc;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001850 __be16 proto = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001851 int err;
Ben Greear3bdc0eb2012-02-11 15:39:30 +00001852 int extra_len = 0;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001853
Linus Torvalds1da177e2005-04-16 15:20:36 -07001854 /*
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001855 * Get and verify the address.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001856 */
1857
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001858 if (saddr) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001859 if (msg->msg_namelen < sizeof(struct sockaddr))
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001860 return -EINVAL;
1861 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1862 proto = saddr->spkt_protocol;
1863 } else
1864 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001865
1866 /*
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001867 * Find the device first to size check it
Linus Torvalds1da177e2005-04-16 15:20:36 -07001868 */
1869
danborkmann@iogearbox.netde74e922012-06-10 08:59:28 +00001870 saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001871retry:
Eric Dumazet654d1f82009-11-02 10:43:32 +01001872 rcu_read_lock();
1873 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001874 err = -ENODEV;
1875 if (dev == NULL)
1876 goto out_unlock;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001877
David S. Millerd5e76b02007-01-25 19:30:36 -08001878 err = -ENETDOWN;
1879 if (!(dev->flags & IFF_UP))
1880 goto out_unlock;
1881
Linus Torvalds1da177e2005-04-16 15:20:36 -07001882 /*
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001883 * You may not queue a frame bigger than the mtu. This is the lowest level
1884 * raw protocol and you must do your own fragmentation at this level.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001885 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001886
Ben Greear3bdc0eb2012-02-11 15:39:30 +00001887 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1888 if (!netif_supports_nofcs(dev)) {
1889 err = -EPROTONOSUPPORT;
1890 goto out_unlock;
1891 }
1892 extra_len = 4; /* We're doing our own CRC */
1893 }
1894
Linus Torvalds1da177e2005-04-16 15:20:36 -07001895 err = -EMSGSIZE;
Ben Greear3bdc0eb2012-02-11 15:39:30 +00001896 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001897 goto out_unlock;
1898
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001899 if (!skb) {
1900 size_t reserved = LL_RESERVED_SPACE(dev);
Herbert Xu4ce40912011-11-18 02:20:05 +00001901 int tlen = dev->needed_tailroom;
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001902 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001903
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001904 rcu_read_unlock();
Herbert Xu4ce40912011-11-18 02:20:05 +00001905 skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001906 if (skb == NULL)
1907 return -ENOBUFS;
1908 /* FIXME: Save some space for broken drivers that write a hard
1909 * header at transmission time by themselves. PPP is the notable
1910 * one here. This should really be fixed at the driver level.
1911 */
1912 skb_reserve(skb, reserved);
1913 skb_reset_network_header(skb);
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001914
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001915 /* Try to align data part correctly */
1916 if (hhlen) {
1917 skb->data -= hhlen;
1918 skb->tail -= hhlen;
1919 if (len < hhlen)
1920 skb_reset_network_header(skb);
1921 }
Al Viro6ce8e9c2014-04-06 21:25:44 -04001922 err = memcpy_from_msg(skb_put(skb, len), msg, len);
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001923 if (err)
1924 goto out_free;
1925 goto retry;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001926 }
1927
Willem de Bruijn9ed988c2016-03-09 21:58:34 -05001928 if (!dev_validate_header(dev, skb->data, len)) {
1929 err = -EINVAL;
1930 goto out_unlock;
1931 }
Daniel Borkmann3c70c132015-11-11 23:25:42 +01001932 if (len > (dev->mtu + dev->hard_header_len + extra_len) &&
1933 !packet_extra_vlan_len_allowed(dev, skb)) {
1934 err = -EMSGSIZE;
1935 goto out_unlock;
Ben Greear57f89bf2011-02-11 09:35:18 +00001936 }
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001937
Yoshihiro Shimodaedbe7742016-07-19 14:40:51 +09001938 sockc.tsflags = sk->sk_tsflags;
Soheil Hassas Yeganehc14ac942016-04-02 23:08:12 -04001939 if (msg->msg_controllen) {
1940 err = sock_cmsg_send(sk, msg, &sockc);
Soheil Hassas Yeganehf8e77182016-07-20 18:01:18 -04001941 if (unlikely(err))
Soheil Hassas Yeganehc14ac942016-04-02 23:08:12 -04001942 goto out_unlock;
Soheil Hassas Yeganehc14ac942016-04-02 23:08:12 -04001943 }
1944
Linus Torvalds1da177e2005-04-16 15:20:36 -07001945 skb->protocol = proto;
1946 skb->dev = dev;
1947 skb->priority = sk->sk_priority;
Eric Dumazet2d37a182009-10-01 19:14:46 +00001948 skb->mark = sk->sk_mark;
Daniel Borkmannbf84a012013-04-14 08:08:13 +00001949
Soheil Hassas Yeganehc14ac942016-04-02 23:08:12 -04001950 sock_tx_timestamp(sk, sockc.tsflags, &skb_shinfo(skb)->tx_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001951
Ben Greear3bdc0eb2012-02-11 15:39:30 +00001952 if (unlikely(extra_len == 4))
1953 skb->no_fcs = 1;
1954
Jason Wang40893fd2013-03-26 23:11:22 +00001955 skb_probe_transport_header(skb, 0);
Jason Wangc1aad272013-03-25 20:19:57 +00001956
Linus Torvalds1da177e2005-04-16 15:20:36 -07001957 dev_queue_xmit(skb);
Eric Dumazet654d1f82009-11-02 10:43:32 +01001958 rcu_read_unlock();
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001959 return len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001960
Linus Torvalds1da177e2005-04-16 15:20:36 -07001961out_unlock:
Eric Dumazet654d1f82009-11-02 10:43:32 +01001962 rcu_read_unlock();
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001963out_free:
1964 kfree_skb(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001965 return err;
1966}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001967
Alexei Starovoitovff936a02015-10-07 10:55:41 -07001968static unsigned int run_filter(struct sk_buff *skb,
1969 const struct sock *sk,
1970 unsigned int res)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001971{
1972 struct sk_filter *filter;
1973
Eric Dumazet80f8f102011-01-18 07:46:52 +00001974 rcu_read_lock();
1975 filter = rcu_dereference(sk->sk_filter);
David S. Millerdbcb5852007-01-24 15:21:02 -08001976 if (filter != NULL)
Alexei Starovoitovff936a02015-10-07 10:55:41 -07001977 res = bpf_prog_run_clear_cb(filter->prog, skb);
Eric Dumazet80f8f102011-01-18 07:46:52 +00001978 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001979
David S. Millerdbcb5852007-01-24 15:21:02 -08001980 return res;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001981}
1982
Willem de Bruijn16cc1402016-02-03 18:02:14 -05001983static int __packet_rcv_vnet(const struct sk_buff *skb,
1984 struct virtio_net_hdr *vnet_hdr)
1985{
1986 *vnet_hdr = (const struct virtio_net_hdr) { 0 };
1987
Jason Wang1e7cbb42017-01-20 14:32:42 +08001988 if (virtio_net_hdr_from_skb(skb, vnet_hdr, vio_le(), true))
Mike Rapoport1276f242016-06-08 16:09:22 +03001989 BUG();
Willem de Bruijn16cc1402016-02-03 18:02:14 -05001990
1991 return 0;
1992}
1993
1994static int packet_rcv_vnet(struct msghdr *msg, const struct sk_buff *skb,
1995 size_t *len)
1996{
1997 struct virtio_net_hdr vnet_hdr;
1998
1999 if (*len < sizeof(vnet_hdr))
2000 return -EINVAL;
2001 *len -= sizeof(vnet_hdr);
2002
2003 if (__packet_rcv_vnet(skb, &vnet_hdr))
2004 return -EINVAL;
2005
2006 return memcpy_to_msg(msg, (void *)&vnet_hdr, sizeof(vnet_hdr));
2007}
2008
Linus Torvalds1da177e2005-04-16 15:20:36 -07002009/*
Eric Dumazet62ab0812010-12-06 20:50:09 +00002010 * This function makes lazy skb cloning in hope that most of packets
2011 * are discarded by BPF.
2012 *
2013 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
2014 * and skb->cb are mangled. It works because (and until) packets
2015 * falling here are owned by current CPU. Output packets are cloned
2016 * by dev_queue_xmit_nit(), input packets are processed by net_bh
2017 * sequencially, so that if we return skb to original state on exit,
2018 * we will not harm anyone.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002019 */
2020
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002021static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
2022 struct packet_type *pt, struct net_device *orig_dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002023{
2024 struct sock *sk;
2025 struct sockaddr_ll *sll;
2026 struct packet_sock *po;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002027 u8 *skb_head = skb->data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002028 int skb_len = skb->len;
David S. Millerdbcb5852007-01-24 15:21:02 -08002029 unsigned int snaplen, res;
Weongyo Jeongda378452016-04-14 14:10:04 -07002030 bool is_drop_n_account = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002031
2032 if (skb->pkt_type == PACKET_LOOPBACK)
2033 goto drop;
2034
2035 sk = pt->af_packet_priv;
2036 po = pkt_sk(sk);
2037
Octavian Purdila09ad9bc2009-11-25 15:14:13 -08002038 if (!net_eq(dev_net(dev), sock_net(sk)))
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08002039 goto drop;
2040
Linus Torvalds1da177e2005-04-16 15:20:36 -07002041 skb->dev = dev;
2042
Stephen Hemminger3b04ddd2007-10-09 01:40:57 -07002043 if (dev->header_ops) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002044 /* The device has an explicit notion of ll header,
Eric Dumazet62ab0812010-12-06 20:50:09 +00002045 * exported to higher levels.
2046 *
2047 * Otherwise, the device hides details of its frame
2048 * structure, so that corresponding packet head is
2049 * never delivered to user.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002050 */
2051 if (sk->sk_type != SOCK_DGRAM)
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -07002052 skb_push(skb, skb->data - skb_mac_header(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002053 else if (skb->pkt_type == PACKET_OUTGOING) {
2054 /* Special case: outgoing packets have ll header at head */
Arnaldo Carvalho de Melobbe735e2007-03-10 22:16:10 -03002055 skb_pull(skb, skb_network_offset(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002056 }
2057 }
2058
2059 snaplen = skb->len;
2060
David S. Millerdbcb5852007-01-24 15:21:02 -08002061 res = run_filter(skb, sk, snaplen);
2062 if (!res)
Dmitry Mishinfda9ef52006-08-31 15:28:39 -07002063 goto drop_n_restore;
David S. Millerdbcb5852007-01-24 15:21:02 -08002064 if (snaplen > res)
2065 snaplen = res;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002066
Eric Dumazet0fd7bac2011-12-21 07:11:44 +00002067 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002068 goto drop_n_acct;
2069
2070 if (skb_shared(skb)) {
2071 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
2072 if (nskb == NULL)
2073 goto drop_n_acct;
2074
2075 if (skb_head != skb->data) {
2076 skb->data = skb_head;
2077 skb->len = skb_len;
2078 }
Eric Dumazetabc4e4f2012-04-19 02:24:42 +00002079 consume_skb(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002080 skb = nskb;
2081 }
2082
Eyal Birgerb4772ef2015-03-01 14:58:29 +02002083 sock_skb_cb_check_size(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8);
Herbert Xuffbc6112007-02-04 23:33:10 -08002084
2085 sll = &PACKET_SKB_CB(skb)->sa.ll;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002086 sll->sll_hatype = dev->type;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002087 sll->sll_pkttype = skb->pkt_type;
Peter P Waskiewicz Jr8032b462007-11-10 22:03:25 -08002088 if (unlikely(po->origdev))
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07002089 sll->sll_ifindex = orig_dev->ifindex;
2090 else
2091 sll->sll_ifindex = dev->ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002092
Stephen Hemmingerb95cce32007-09-26 22:13:38 -07002093 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002094
Eyal Birger2472d762015-03-01 14:58:28 +02002095 /* sll->sll_family and sll->sll_protocol are set in packet_recvmsg().
2096 * Use their space for storing the original skb length.
2097 */
2098 PACKET_SKB_CB(skb)->sa.origlen = skb->len;
Herbert Xu8dc41942007-02-04 23:31:32 -08002099
Linus Torvalds1da177e2005-04-16 15:20:36 -07002100 if (pskb_trim(skb, snaplen))
2101 goto drop_n_acct;
2102
2103 skb_set_owner_r(skb, sk);
2104 skb->dev = NULL;
Eric Dumazetadf30902009-06-02 05:19:30 +00002105 skb_dst_drop(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002106
Phil Oester84531c22005-07-12 11:57:52 -07002107 /* drop conntrack reference */
2108 nf_reset(skb);
2109
Linus Torvalds1da177e2005-04-16 15:20:36 -07002110 spin_lock(&sk->sk_receive_queue.lock);
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00002111 po->stats.stats1.tp_packets++;
Eyal Birger3bc3b962015-03-01 14:58:30 +02002112 sock_skb_set_dropcount(sk, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002113 __skb_queue_tail(&sk->sk_receive_queue, skb);
2114 spin_unlock(&sk->sk_receive_queue.lock);
David S. Miller676d2362014-04-11 16:15:36 -04002115 sk->sk_data_ready(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002116 return 0;
2117
2118drop_n_acct:
Weongyo Jeongda378452016-04-14 14:10:04 -07002119 is_drop_n_account = true;
Willem de Bruijn7091fbd2011-09-30 10:38:28 +00002120 spin_lock(&sk->sk_receive_queue.lock);
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00002121 po->stats.stats1.tp_drops++;
Willem de Bruijn7091fbd2011-09-30 10:38:28 +00002122 atomic_inc(&sk->sk_drops);
2123 spin_unlock(&sk->sk_receive_queue.lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002124
2125drop_n_restore:
2126 if (skb_head != skb->data && skb_shared(skb)) {
2127 skb->data = skb_head;
2128 skb->len = skb_len;
2129 }
2130drop:
Weongyo Jeongda378452016-04-14 14:10:04 -07002131 if (!is_drop_n_account)
2132 consume_skb(skb);
2133 else
2134 kfree_skb(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002135 return 0;
2136}
2137
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002138static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
2139 struct packet_type *pt, struct net_device *orig_dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002140{
2141 struct sock *sk;
2142 struct packet_sock *po;
2143 struct sockaddr_ll *sll;
Daniel Borkmann184f4892013-04-16 01:57:46 +00002144 union tpacket_uhdr h;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002145 u8 *skb_head = skb->data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002146 int skb_len = skb->len;
David S. Millerdbcb5852007-01-24 15:21:02 -08002147 unsigned int snaplen, res;
chetan lokef6fb8f102011-08-19 10:18:16 +00002148 unsigned long status = TP_STATUS_USER;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07002149 unsigned short macoff, netoff, hdrlen;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002150 struct sk_buff *copy_skb = NULL;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07002151 struct timespec ts;
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +00002152 __u32 ts_status;
Weongyo Jeongda378452016-04-14 14:10:04 -07002153 bool is_drop_n_account = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002154
Atzm Watanabe51846352013-12-17 22:53:32 +09002155 /* struct tpacket{2,3}_hdr is aligned to a multiple of TPACKET_ALIGNMENT.
2156 * We may add members to them until current aligned size without forcing
2157 * userspace to call getsockopt(..., PACKET_HDRLEN, ...).
2158 */
2159 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h2)) != 32);
2160 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h3)) != 48);
2161
Linus Torvalds1da177e2005-04-16 15:20:36 -07002162 if (skb->pkt_type == PACKET_LOOPBACK)
2163 goto drop;
2164
2165 sk = pt->af_packet_priv;
2166 po = pkt_sk(sk);
2167
Octavian Purdila09ad9bc2009-11-25 15:14:13 -08002168 if (!net_eq(dev_net(dev), sock_net(sk)))
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08002169 goto drop;
2170
Stephen Hemminger3b04ddd2007-10-09 01:40:57 -07002171 if (dev->header_ops) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002172 if (sk->sk_type != SOCK_DGRAM)
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -07002173 skb_push(skb, skb->data - skb_mac_header(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002174 else if (skb->pkt_type == PACKET_OUTGOING) {
2175 /* Special case: outgoing packets have ll header at head */
Arnaldo Carvalho de Melobbe735e2007-03-10 22:16:10 -03002176 skb_pull(skb, skb_network_offset(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002177 }
2178 }
2179
2180 snaplen = skb->len;
2181
David S. Millerdbcb5852007-01-24 15:21:02 -08002182 res = run_filter(skb, sk, snaplen);
2183 if (!res)
Dmitry Mishinfda9ef52006-08-31 15:28:39 -07002184 goto drop_n_restore;
Alexander Drozdov68c2e5d2015-03-23 09:11:12 +03002185
2186 if (skb->ip_summed == CHECKSUM_PARTIAL)
2187 status |= TP_STATUS_CSUMNOTREADY;
Alexander Drozdov682f0482015-03-23 09:11:13 +03002188 else if (skb->pkt_type != PACKET_OUTGOING &&
2189 (skb->ip_summed == CHECKSUM_COMPLETE ||
2190 skb_csum_unnecessary(skb)))
2191 status |= TP_STATUS_CSUM_VALID;
Alexander Drozdov68c2e5d2015-03-23 09:11:12 +03002192
David S. Millerdbcb5852007-01-24 15:21:02 -08002193 if (snaplen > res)
2194 snaplen = res;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002195
2196 if (sk->sk_type == SOCK_DGRAM) {
Patrick McHardy8913336a2008-07-18 18:05:19 -07002197 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
2198 po->tp_reserve;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002199 } else {
Eric Dumazet95c96172012-04-15 05:58:06 +00002200 unsigned int maclen = skb_network_offset(skb);
Patrick McHardybbd6ef82008-07-14 22:50:15 -07002201 netoff = TPACKET_ALIGN(po->tp_hdrlen +
Patrick McHardy8913336a2008-07-18 18:05:19 -07002202 (maclen < 16 ? 16 : maclen)) +
Willem de Bruijn58d19b12016-02-03 18:02:15 -05002203 po->tp_reserve;
2204 if (po->has_vnet_hdr)
2205 netoff += sizeof(struct virtio_net_hdr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002206 macoff = netoff - maclen;
2207 }
chetan lokef6fb8f102011-08-19 10:18:16 +00002208 if (po->tp_version <= TPACKET_V2) {
2209 if (macoff + snaplen > po->rx_ring.frame_size) {
2210 if (po->copy_thresh &&
Eric Dumazet0fd7bac2011-12-21 07:11:44 +00002211 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
chetan lokef6fb8f102011-08-19 10:18:16 +00002212 if (skb_shared(skb)) {
2213 copy_skb = skb_clone(skb, GFP_ATOMIC);
2214 } else {
2215 copy_skb = skb_get(skb);
2216 skb_head = skb->data;
2217 }
2218 if (copy_skb)
2219 skb_set_owner_r(copy_skb, sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002220 }
chetan lokef6fb8f102011-08-19 10:18:16 +00002221 snaplen = po->rx_ring.frame_size - macoff;
2222 if ((int)snaplen < 0)
2223 snaplen = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002224 }
Eric Dumazetdc808112014-08-15 09:16:04 -07002225 } else if (unlikely(macoff + snaplen >
2226 GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len)) {
2227 u32 nval;
2228
2229 nval = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len - macoff;
2230 pr_err_once("tpacket_rcv: packet too big, clamped from %u to %u. macoff=%u\n",
2231 snaplen, nval, macoff);
2232 snaplen = nval;
2233 if (unlikely((int)snaplen < 0)) {
2234 snaplen = 0;
2235 macoff = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len;
2236 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002237 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002238 spin_lock(&sk->sk_receive_queue.lock);
chetan lokef6fb8f102011-08-19 10:18:16 +00002239 h.raw = packet_current_rx_frame(po, skb,
2240 TP_STATUS_KERNEL, (macoff+snaplen));
Patrick McHardybbd6ef82008-07-14 22:50:15 -07002241 if (!h.raw)
Willem de Bruijn58d19b12016-02-03 18:02:15 -05002242 goto drop_n_account;
chetan lokef6fb8f102011-08-19 10:18:16 +00002243 if (po->tp_version <= TPACKET_V2) {
2244 packet_increment_rx_head(po, &po->rx_ring);
2245 /*
2246 * LOSING will be reported till you read the stats,
2247 * because it's COR - Clear On Read.
2248 * Anyways, moving it for V1/V2 only as V3 doesn't need this
2249 * at packet level.
2250 */
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00002251 if (po->stats.stats1.tp_drops)
chetan lokef6fb8f102011-08-19 10:18:16 +00002252 status |= TP_STATUS_LOSING;
2253 }
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00002254 po->stats.stats1.tp_packets++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002255 if (copy_skb) {
2256 status |= TP_STATUS_COPY;
2257 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
2258 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002259 spin_unlock(&sk->sk_receive_queue.lock);
2260
Willem de Bruijn58d19b12016-02-03 18:02:15 -05002261 if (po->has_vnet_hdr) {
2262 if (__packet_rcv_vnet(skb, h.raw + macoff -
2263 sizeof(struct virtio_net_hdr))) {
2264 spin_lock(&sk->sk_receive_queue.lock);
2265 goto drop_n_account;
2266 }
2267 }
2268
Patrick McHardybbd6ef82008-07-14 22:50:15 -07002269 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +00002270
2271 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
Daniel Borkmann7a513842013-04-23 00:39:29 +00002272 getnstimeofday(&ts);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002273
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +00002274 status |= ts_status;
2275
Patrick McHardybbd6ef82008-07-14 22:50:15 -07002276 switch (po->tp_version) {
2277 case TPACKET_V1:
2278 h.h1->tp_len = skb->len;
2279 h.h1->tp_snaplen = snaplen;
2280 h.h1->tp_mac = macoff;
2281 h.h1->tp_net = netoff;
Daniel Borkmann4b457bd2013-04-16 01:29:11 +00002282 h.h1->tp_sec = ts.tv_sec;
2283 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07002284 hdrlen = sizeof(*h.h1);
2285 break;
2286 case TPACKET_V2:
2287 h.h2->tp_len = skb->len;
2288 h.h2->tp_snaplen = snaplen;
2289 h.h2->tp_mac = macoff;
2290 h.h2->tp_net = netoff;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07002291 h.h2->tp_sec = ts.tv_sec;
2292 h.h2->tp_nsec = ts.tv_nsec;
Jiri Pirkodf8a39d2015-01-13 17:13:44 +01002293 if (skb_vlan_tag_present(skb)) {
2294 h.h2->tp_vlan_tci = skb_vlan_tag_get(skb);
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +09002295 h.h2->tp_vlan_tpid = ntohs(skb->vlan_proto);
2296 status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
Ben Greeara3bcc232011-06-01 06:49:10 +00002297 } else {
2298 h.h2->tp_vlan_tci = 0;
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +09002299 h.h2->tp_vlan_tpid = 0;
Ben Greeara3bcc232011-06-01 06:49:10 +00002300 }
Atzm Watanabee4d26f42013-12-17 22:53:36 +09002301 memset(h.h2->tp_padding, 0, sizeof(h.h2->tp_padding));
Patrick McHardybbd6ef82008-07-14 22:50:15 -07002302 hdrlen = sizeof(*h.h2);
2303 break;
chetan lokef6fb8f102011-08-19 10:18:16 +00002304 case TPACKET_V3:
2305 /* tp_nxt_offset,vlan are already populated above.
2306 * So DONT clear those fields here
2307 */
2308 h.h3->tp_status |= status;
2309 h.h3->tp_len = skb->len;
2310 h.h3->tp_snaplen = snaplen;
2311 h.h3->tp_mac = macoff;
2312 h.h3->tp_net = netoff;
chetan lokef6fb8f102011-08-19 10:18:16 +00002313 h.h3->tp_sec = ts.tv_sec;
2314 h.h3->tp_nsec = ts.tv_nsec;
Atzm Watanabee4d26f42013-12-17 22:53:36 +09002315 memset(h.h3->tp_padding, 0, sizeof(h.h3->tp_padding));
chetan lokef6fb8f102011-08-19 10:18:16 +00002316 hdrlen = sizeof(*h.h3);
2317 break;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07002318 default:
2319 BUG();
2320 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002321
Patrick McHardybbd6ef82008-07-14 22:50:15 -07002322 sll = h.raw + TPACKET_ALIGN(hdrlen);
Stephen Hemmingerb95cce32007-09-26 22:13:38 -07002323 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002324 sll->sll_family = AF_PACKET;
2325 sll->sll_hatype = dev->type;
2326 sll->sll_protocol = skb->protocol;
2327 sll->sll_pkttype = skb->pkt_type;
Peter P Waskiewicz Jr8032b462007-11-10 22:03:25 -08002328 if (unlikely(po->origdev))
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07002329 sll->sll_ifindex = orig_dev->ifindex;
2330 else
2331 sll->sll_ifindex = dev->ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002332
Ralf Baechlee16aa202006-12-07 00:11:33 -08002333 smp_mb();
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01002334
Changli Gaof6dafa92010-12-07 04:26:16 +00002335#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01002336 if (po->tp_version <= TPACKET_V2) {
Changli Gao0af55bb2010-12-01 02:52:20 +00002337 u8 *start, *end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002338
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01002339 end = (u8 *) PAGE_ALIGN((unsigned long) h.raw +
2340 macoff + snaplen);
2341
2342 for (start = h.raw; start < end; start += PAGE_SIZE)
2343 flush_dcache_page(pgv_to_page(start));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002344 }
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01002345 smp_wmb();
Changli Gaof6dafa92010-12-07 04:26:16 +00002346#endif
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01002347
Dan Collinsda413ee2014-12-19 16:49:25 +13002348 if (po->tp_version <= TPACKET_V2) {
chetan lokef6fb8f102011-08-19 10:18:16 +00002349 __packet_set_status(po, h.raw, status);
Dan Collinsda413ee2014-12-19 16:49:25 +13002350 sk->sk_data_ready(sk);
2351 } else {
chetan lokef6fb8f102011-08-19 10:18:16 +00002352 prb_clear_blk_fill_status(&po->rx_ring);
Dan Collinsda413ee2014-12-19 16:49:25 +13002353 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002354
2355drop_n_restore:
2356 if (skb_head != skb->data && skb_shared(skb)) {
2357 skb->data = skb_head;
2358 skb->len = skb_len;
2359 }
2360drop:
Weongyo Jeongda378452016-04-14 14:10:04 -07002361 if (!is_drop_n_account)
2362 consume_skb(skb);
2363 else
2364 kfree_skb(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002365 return 0;
2366
Willem de Bruijn58d19b12016-02-03 18:02:15 -05002367drop_n_account:
Weongyo Jeongda378452016-04-14 14:10:04 -07002368 is_drop_n_account = true;
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00002369 po->stats.stats1.tp_drops++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002370 spin_unlock(&sk->sk_receive_queue.lock);
2371
David S. Miller676d2362014-04-11 16:15:36 -04002372 sk->sk_data_ready(sk);
Wei Yongjunacb5d752009-02-25 00:36:42 +00002373 kfree_skb(copy_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002374 goto drop_n_restore;
2375}
2376
Johann Baudy69e3c752009-05-18 22:11:22 -07002377static void tpacket_destruct_skb(struct sk_buff *skb)
2378{
2379 struct packet_sock *po = pkt_sk(skb->sk);
Johann Baudy69e3c752009-05-18 22:11:22 -07002380
Johann Baudy69e3c752009-05-18 22:11:22 -07002381 if (likely(po->tx_ring.pg_vec)) {
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01002382 void *ph;
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +00002383 __u32 ts;
2384
Johann Baudy69e3c752009-05-18 22:11:22 -07002385 ph = skb_shinfo(skb)->destructor_arg;
Daniel Borkmannb0138402014-01-15 16:25:36 +01002386 packet_dec_pending(&po->tx_ring);
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +00002387
2388 ts = __packet_set_timestamp(po, ph, skb);
2389 __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
Johann Baudy69e3c752009-05-18 22:11:22 -07002390 }
2391
2392 sock_wfree(skb);
2393}
2394
Daniel Borkmannc72219b2015-11-11 23:25:43 +01002395static void tpacket_set_protocol(const struct net_device *dev,
2396 struct sk_buff *skb)
2397{
2398 if (dev->type == ARPHRD_ETHER) {
2399 skb_reset_mac_header(skb);
2400 skb->protocol = eth_hdr(skb)->h_proto;
2401 }
2402}
2403
Willem de Bruijn16cc1402016-02-03 18:02:14 -05002404static int __packet_snd_vnet_parse(struct virtio_net_hdr *vnet_hdr, size_t len)
2405{
2406 unsigned short gso_type = 0;
2407
2408 if ((vnet_hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
2409 (__virtio16_to_cpu(vio_le(), vnet_hdr->csum_start) +
2410 __virtio16_to_cpu(vio_le(), vnet_hdr->csum_offset) + 2 >
2411 __virtio16_to_cpu(vio_le(), vnet_hdr->hdr_len)))
2412 vnet_hdr->hdr_len = __cpu_to_virtio16(vio_le(),
2413 __virtio16_to_cpu(vio_le(), vnet_hdr->csum_start) +
2414 __virtio16_to_cpu(vio_le(), vnet_hdr->csum_offset) + 2);
2415
2416 if (__virtio16_to_cpu(vio_le(), vnet_hdr->hdr_len) > len)
2417 return -EINVAL;
2418
2419 if (vnet_hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) {
2420 switch (vnet_hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2421 case VIRTIO_NET_HDR_GSO_TCPV4:
2422 gso_type = SKB_GSO_TCPV4;
2423 break;
2424 case VIRTIO_NET_HDR_GSO_TCPV6:
2425 gso_type = SKB_GSO_TCPV6;
2426 break;
2427 case VIRTIO_NET_HDR_GSO_UDP:
2428 gso_type = SKB_GSO_UDP;
2429 break;
2430 default:
2431 return -EINVAL;
2432 }
2433
2434 if (vnet_hdr->gso_type & VIRTIO_NET_HDR_GSO_ECN)
2435 gso_type |= SKB_GSO_TCP_ECN;
2436
2437 if (vnet_hdr->gso_size == 0)
2438 return -EINVAL;
2439 }
2440
2441 vnet_hdr->gso_type = gso_type; /* changes type, temporary storage */
2442 return 0;
2443}
2444
2445static int packet_snd_vnet_parse(struct msghdr *msg, size_t *len,
2446 struct virtio_net_hdr *vnet_hdr)
2447{
2448 int n;
2449
2450 if (*len < sizeof(*vnet_hdr))
2451 return -EINVAL;
2452 *len -= sizeof(*vnet_hdr);
2453
2454 n = copy_from_iter(vnet_hdr, sizeof(*vnet_hdr), &msg->msg_iter);
2455 if (n != sizeof(*vnet_hdr))
2456 return -EFAULT;
2457
2458 return __packet_snd_vnet_parse(vnet_hdr, *len);
2459}
2460
2461static int packet_snd_vnet_gso(struct sk_buff *skb,
2462 struct virtio_net_hdr *vnet_hdr)
2463{
2464 if (vnet_hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
2465 u16 s = __virtio16_to_cpu(vio_le(), vnet_hdr->csum_start);
2466 u16 o = __virtio16_to_cpu(vio_le(), vnet_hdr->csum_offset);
2467
2468 if (!skb_partial_csum_set(skb, s, o))
2469 return -EINVAL;
2470 }
2471
2472 skb_shinfo(skb)->gso_size =
2473 __virtio16_to_cpu(vio_le(), vnet_hdr->gso_size);
2474 skb_shinfo(skb)->gso_type = vnet_hdr->gso_type;
2475
2476 /* Header must be checked, and gso_segs computed. */
2477 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2478 skb_shinfo(skb)->gso_segs = 0;
2479 return 0;
2480}
2481
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002482static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
Willem de Bruijn8d39b4a2016-02-03 18:02:16 -05002483 void *frame, struct net_device *dev, void *data, int tp_len,
Soheil Hassas Yeganehc14ac942016-04-02 23:08:12 -04002484 __be16 proto, unsigned char *addr, int hlen, int copylen,
2485 const struct sockcm_cookie *sockc)
Johann Baudy69e3c752009-05-18 22:11:22 -07002486{
Daniel Borkmann184f4892013-04-16 01:57:46 +00002487 union tpacket_uhdr ph;
Willem de Bruijn8d39b4a2016-02-03 18:02:16 -05002488 int to_write, offset, len, nr_frags, len_max;
Johann Baudy69e3c752009-05-18 22:11:22 -07002489 struct socket *sock = po->sk.sk_socket;
2490 struct page *page;
Johann Baudy69e3c752009-05-18 22:11:22 -07002491 int err;
2492
2493 ph.raw = frame;
2494
2495 skb->protocol = proto;
2496 skb->dev = dev;
2497 skb->priority = po->sk.sk_priority;
Eric Dumazet2d37a182009-10-01 19:14:46 +00002498 skb->mark = po->sk.sk_mark;
Soheil Hassas Yeganehc14ac942016-04-02 23:08:12 -04002499 sock_tx_timestamp(&po->sk, sockc->tsflags, &skb_shinfo(skb)->tx_flags);
Johann Baudy69e3c752009-05-18 22:11:22 -07002500 skb_shinfo(skb)->destructor_arg = ph.raw;
2501
Herbert Xuae641942011-11-18 02:20:04 +00002502 skb_reserve(skb, hlen);
Johann Baudy69e3c752009-05-18 22:11:22 -07002503 skb_reset_network_header(skb);
Jason Wangc1aad272013-03-25 20:19:57 +00002504
Johann Baudy69e3c752009-05-18 22:11:22 -07002505 to_write = tp_len;
2506
2507 if (sock->type == SOCK_DGRAM) {
2508 err = dev_hard_header(skb, dev, ntohs(proto), addr,
2509 NULL, tp_len);
2510 if (unlikely(err < 0))
2511 return -EINVAL;
Willem de Bruijn1d036d22016-02-03 18:02:17 -05002512 } else if (copylen) {
Willem de Bruijn9ed988c2016-03-09 21:58:34 -05002513 int hdrlen = min_t(int, copylen, tp_len);
2514
Johann Baudy69e3c752009-05-18 22:11:22 -07002515 skb_push(skb, dev->hard_header_len);
Willem de Bruijn1d036d22016-02-03 18:02:17 -05002516 skb_put(skb, copylen - dev->hard_header_len);
Willem de Bruijn9ed988c2016-03-09 21:58:34 -05002517 err = skb_store_bits(skb, 0, data, hdrlen);
Johann Baudy69e3c752009-05-18 22:11:22 -07002518 if (unlikely(err))
2519 return err;
Willem de Bruijn9ed988c2016-03-09 21:58:34 -05002520 if (!dev_validate_header(dev, skb->data, hdrlen))
2521 return -EINVAL;
Daniel Borkmannc72219b2015-11-11 23:25:43 +01002522 if (!skb->protocol)
2523 tpacket_set_protocol(dev, skb);
Johann Baudy69e3c752009-05-18 22:11:22 -07002524
Willem de Bruijn9ed988c2016-03-09 21:58:34 -05002525 data += hdrlen;
2526 to_write -= hdrlen;
Johann Baudy69e3c752009-05-18 22:11:22 -07002527 }
2528
Johann Baudy69e3c752009-05-18 22:11:22 -07002529 offset = offset_in_page(data);
2530 len_max = PAGE_SIZE - offset;
2531 len = ((to_write > len_max) ? len_max : to_write);
2532
2533 skb->data_len = to_write;
2534 skb->len += to_write;
2535 skb->truesize += to_write;
2536 atomic_add(to_write, &po->sk.sk_wmem_alloc);
2537
2538 while (likely(to_write)) {
2539 nr_frags = skb_shinfo(skb)->nr_frags;
2540
2541 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002542 pr_err("Packet exceed the number of skb frags(%lu)\n",
2543 MAX_SKB_FRAGS);
Johann Baudy69e3c752009-05-18 22:11:22 -07002544 return -EFAULT;
2545 }
2546
Changli Gao0af55bb2010-12-01 02:52:20 +00002547 page = pgv_to_page(data);
2548 data += len;
Johann Baudy69e3c752009-05-18 22:11:22 -07002549 flush_dcache_page(page);
2550 get_page(page);
Changli Gao0af55bb2010-12-01 02:52:20 +00002551 skb_fill_page_desc(skb, nr_frags, page, offset, len);
Johann Baudy69e3c752009-05-18 22:11:22 -07002552 to_write -= len;
2553 offset = 0;
2554 len_max = PAGE_SIZE;
2555 len = ((to_write > len_max) ? len_max : to_write);
2556 }
2557
Daniel Borkmann8fd6c802015-11-11 23:25:41 +01002558 skb_probe_transport_header(skb, 0);
Daniel Borkmannefdfa2f2015-11-11 23:25:40 +01002559
Johann Baudy69e3c752009-05-18 22:11:22 -07002560 return tp_len;
2561}
2562
Willem de Bruijn8d39b4a2016-02-03 18:02:16 -05002563static int tpacket_parse_header(struct packet_sock *po, void *frame,
2564 int size_max, void **data)
2565{
2566 union tpacket_uhdr ph;
2567 int tp_len, off;
2568
2569 ph.raw = frame;
2570
2571 switch (po->tp_version) {
2572 case TPACKET_V2:
2573 tp_len = ph.h2->tp_len;
2574 break;
2575 default:
2576 tp_len = ph.h1->tp_len;
2577 break;
2578 }
2579 if (unlikely(tp_len > size_max)) {
2580 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
2581 return -EMSGSIZE;
2582 }
2583
2584 if (unlikely(po->tp_tx_has_off)) {
2585 int off_min, off_max;
2586
2587 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2588 off_max = po->tx_ring.frame_size - tp_len;
2589 if (po->sk.sk_type == SOCK_DGRAM) {
2590 switch (po->tp_version) {
2591 case TPACKET_V2:
2592 off = ph.h2->tp_net;
2593 break;
2594 default:
2595 off = ph.h1->tp_net;
2596 break;
2597 }
2598 } else {
2599 switch (po->tp_version) {
2600 case TPACKET_V2:
2601 off = ph.h2->tp_mac;
2602 break;
2603 default:
2604 off = ph.h1->tp_mac;
2605 break;
2606 }
2607 }
2608 if (unlikely((off < off_min) || (off_max < off)))
2609 return -EINVAL;
2610 } else {
2611 off = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2612 }
2613
2614 *data = frame + off;
2615 return tp_len;
2616}
2617
Johann Baudy69e3c752009-05-18 22:11:22 -07002618static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2619{
Johann Baudy69e3c752009-05-18 22:11:22 -07002620 struct sk_buff *skb;
2621 struct net_device *dev;
Willem de Bruijn1d036d22016-02-03 18:02:17 -05002622 struct virtio_net_hdr *vnet_hdr = NULL;
Soheil Hassas Yeganehc14ac942016-04-02 23:08:12 -04002623 struct sockcm_cookie sockc;
Johann Baudy69e3c752009-05-18 22:11:22 -07002624 __be16 proto;
David S. Miller09effa62013-08-07 17:11:00 -07002625 int err, reserve = 0;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002626 void *ph;
Steffen Hurrle342dfc32014-01-17 22:53:15 +01002627 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
Daniel Borkmann87a2fd22014-01-15 16:25:35 +01002628 bool need_wait = !(msg->msg_flags & MSG_DONTWAIT);
Johann Baudy69e3c752009-05-18 22:11:22 -07002629 int tp_len, size_max;
2630 unsigned char *addr;
Willem de Bruijn8d39b4a2016-02-03 18:02:16 -05002631 void *data;
Johann Baudy69e3c752009-05-18 22:11:22 -07002632 int len_sum = 0;
danborkmann@iogearbox.net9e670302012-08-20 03:34:03 +00002633 int status = TP_STATUS_AVAILABLE;
Willem de Bruijn1d036d22016-02-03 18:02:17 -05002634 int hlen, tlen, copylen = 0;
Johann Baudy69e3c752009-05-18 22:11:22 -07002635
Johann Baudy69e3c752009-05-18 22:11:22 -07002636 mutex_lock(&po->pg_vec_lock);
2637
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01002638 if (likely(saddr == NULL)) {
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002639 dev = packet_cached_dev_get(po);
Johann Baudy69e3c752009-05-18 22:11:22 -07002640 proto = po->num;
2641 addr = NULL;
2642 } else {
2643 err = -EINVAL;
2644 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2645 goto out;
2646 if (msg->msg_namelen < (saddr->sll_halen
2647 + offsetof(struct sockaddr_ll,
2648 sll_addr)))
2649 goto out;
Johann Baudy69e3c752009-05-18 22:11:22 -07002650 proto = saddr->sll_protocol;
2651 addr = saddr->sll_addr;
Ben Greear827d9782011-06-01 07:18:53 +00002652 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
Johann Baudy69e3c752009-05-18 22:11:22 -07002653 }
2654
Johann Baudy69e3c752009-05-18 22:11:22 -07002655 err = -ENXIO;
2656 if (unlikely(dev == NULL))
2657 goto out;
Johann Baudy69e3c752009-05-18 22:11:22 -07002658 err = -ENETDOWN;
2659 if (unlikely(!(dev->flags & IFF_UP)))
2660 goto out_put;
2661
Douglas Caetano dos Santosf79d3302017-05-12 15:19:15 -03002662 sockc.tsflags = po->sk.sk_tsflags;
2663 if (msg->msg_controllen) {
2664 err = sock_cmsg_send(&po->sk, msg, &sockc);
2665 if (unlikely(err))
2666 goto out_put;
2667 }
2668
Daniel Borkmann5cfb4c82015-11-11 23:25:44 +01002669 if (po->sk.sk_socket->type == SOCK_RAW)
2670 reserve = dev->hard_header_len;
Johann Baudy69e3c752009-05-18 22:11:22 -07002671 size_max = po->tx_ring.frame_size
Gabor Gombasb5dd8842009-10-29 03:19:11 -07002672 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
Johann Baudy69e3c752009-05-18 22:11:22 -07002673
Willem de Bruijn1d036d22016-02-03 18:02:17 -05002674 if ((size_max > dev->mtu + reserve + VLAN_HLEN) && !po->has_vnet_hdr)
Daniel Borkmann5cfb4c82015-11-11 23:25:44 +01002675 size_max = dev->mtu + reserve + VLAN_HLEN;
David S. Miller09effa62013-08-07 17:11:00 -07002676
Johann Baudy69e3c752009-05-18 22:11:22 -07002677 do {
2678 ph = packet_current_frame(po, &po->tx_ring,
Daniel Borkmann87a2fd22014-01-15 16:25:35 +01002679 TP_STATUS_SEND_REQUEST);
Johann Baudy69e3c752009-05-18 22:11:22 -07002680 if (unlikely(ph == NULL)) {
Daniel Borkmann87a2fd22014-01-15 16:25:35 +01002681 if (need_wait && need_resched())
2682 schedule();
Johann Baudy69e3c752009-05-18 22:11:22 -07002683 continue;
2684 }
2685
Willem de Bruijn8d39b4a2016-02-03 18:02:16 -05002686 skb = NULL;
2687 tp_len = tpacket_parse_header(po, ph, size_max, &data);
2688 if (tp_len < 0)
2689 goto tpacket_error;
2690
Johann Baudy69e3c752009-05-18 22:11:22 -07002691 status = TP_STATUS_SEND_REQUEST;
Herbert Xuae641942011-11-18 02:20:04 +00002692 hlen = LL_RESERVED_SPACE(dev);
2693 tlen = dev->needed_tailroom;
Willem de Bruijn1d036d22016-02-03 18:02:17 -05002694 if (po->has_vnet_hdr) {
2695 vnet_hdr = data;
2696 data += sizeof(*vnet_hdr);
2697 tp_len -= sizeof(*vnet_hdr);
2698 if (tp_len < 0 ||
2699 __packet_snd_vnet_parse(vnet_hdr, tp_len)) {
2700 tp_len = -EINVAL;
2701 goto tpacket_error;
2702 }
2703 copylen = __virtio16_to_cpu(vio_le(),
2704 vnet_hdr->hdr_len);
2705 }
Willem de Bruijn9ed988c2016-03-09 21:58:34 -05002706 copylen = max_t(int, copylen, dev->hard_header_len);
Johann Baudy69e3c752009-05-18 22:11:22 -07002707 skb = sock_alloc_send_skb(&po->sk,
Willem de Bruijn1d036d22016-02-03 18:02:17 -05002708 hlen + tlen + sizeof(struct sockaddr_ll) +
2709 (copylen - dev->hard_header_len),
Kretschmer, Mathiasfbf33a22015-05-08 15:44:37 +02002710 !need_wait, &err);
Johann Baudy69e3c752009-05-18 22:11:22 -07002711
Kretschmer, Mathiasfbf33a22015-05-08 15:44:37 +02002712 if (unlikely(skb == NULL)) {
2713 /* we assume the socket was initially writeable ... */
2714 if (likely(len_sum > 0))
2715 err = len_sum;
Johann Baudy69e3c752009-05-18 22:11:22 -07002716 goto out_status;
Kretschmer, Mathiasfbf33a22015-05-08 15:44:37 +02002717 }
Willem de Bruijn8d39b4a2016-02-03 18:02:16 -05002718 tp_len = tpacket_fill_skb(po, skb, ph, dev, data, tp_len, proto,
Soheil Hassas Yeganehc14ac942016-04-02 23:08:12 -04002719 addr, hlen, copylen, &sockc);
Alexander Drozdovdbd46ab2015-07-28 13:57:01 +03002720 if (likely(tp_len >= 0) &&
Daniel Borkmann5cfb4c82015-11-11 23:25:44 +01002721 tp_len > dev->mtu + reserve &&
Willem de Bruijn1d036d22016-02-03 18:02:17 -05002722 !po->has_vnet_hdr &&
Daniel Borkmann3c70c132015-11-11 23:25:42 +01002723 !packet_extra_vlan_len_allowed(dev, skb))
2724 tp_len = -EMSGSIZE;
Johann Baudy69e3c752009-05-18 22:11:22 -07002725
2726 if (unlikely(tp_len < 0)) {
Willem de Bruijn8d39b4a2016-02-03 18:02:16 -05002727tpacket_error:
Johann Baudy69e3c752009-05-18 22:11:22 -07002728 if (po->tp_loss) {
2729 __packet_set_status(po, ph,
2730 TP_STATUS_AVAILABLE);
2731 packet_increment_head(&po->tx_ring);
2732 kfree_skb(skb);
2733 continue;
2734 } else {
2735 status = TP_STATUS_WRONG_FORMAT;
2736 err = tp_len;
2737 goto out_status;
2738 }
2739 }
2740
Willem de Bruijn1d036d22016-02-03 18:02:17 -05002741 if (po->has_vnet_hdr && packet_snd_vnet_gso(skb, vnet_hdr)) {
2742 tp_len = -EINVAL;
2743 goto tpacket_error;
2744 }
2745
Daniel Borkmann0fd5d572014-02-16 15:55:22 +01002746 packet_pick_tx_queue(dev, skb);
2747
Johann Baudy69e3c752009-05-18 22:11:22 -07002748 skb->destructor = tpacket_destruct_skb;
2749 __packet_set_status(po, ph, TP_STATUS_SENDING);
Daniel Borkmannb0138402014-01-15 16:25:36 +01002750 packet_inc_pending(&po->tx_ring);
Johann Baudy69e3c752009-05-18 22:11:22 -07002751
2752 status = TP_STATUS_SEND_REQUEST;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01002753 err = po->xmit(skb);
Jarek Poplawskieb70df12010-01-10 22:04:19 +00002754 if (unlikely(err > 0)) {
2755 err = net_xmit_errno(err);
2756 if (err && __packet_get_status(po, ph) ==
2757 TP_STATUS_AVAILABLE) {
2758 /* skb was destructed already */
2759 skb = NULL;
2760 goto out_status;
2761 }
2762 /*
2763 * skb was dropped but not destructed yet;
2764 * let's treat it like congestion or err < 0
2765 */
2766 err = 0;
2767 }
Johann Baudy69e3c752009-05-18 22:11:22 -07002768 packet_increment_head(&po->tx_ring);
2769 len_sum += tp_len;
Daniel Borkmannb0138402014-01-15 16:25:36 +01002770 } while (likely((ph != NULL) ||
2771 /* Note: packet_read_pending() might be slow if we have
2772 * to call it as it's per_cpu variable, but in fast-path
2773 * we already short-circuit the loop with the first
2774 * condition, and luckily don't have to go that path
2775 * anyway.
2776 */
2777 (need_wait && packet_read_pending(&po->tx_ring))));
Johann Baudy69e3c752009-05-18 22:11:22 -07002778
2779 err = len_sum;
2780 goto out_put;
2781
Johann Baudy69e3c752009-05-18 22:11:22 -07002782out_status:
2783 __packet_set_status(po, ph, status);
2784 kfree_skb(skb);
2785out_put:
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002786 dev_put(dev);
Johann Baudy69e3c752009-05-18 22:11:22 -07002787out:
2788 mutex_unlock(&po->pg_vec_lock);
2789 return err;
2790}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002791
Olof Johanssoneea49cc92011-11-02 11:00:49 +00002792static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2793 size_t reserve, size_t len,
2794 size_t linear, int noblock,
2795 int *err)
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002796{
2797 struct sk_buff *skb;
2798
2799 /* Under a page? Don't bother with paged skb. */
2800 if (prepad + len < PAGE_SIZE || !linear)
2801 linear = len;
2802
2803 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
Eric Dumazet28d64272013-08-08 14:38:47 -07002804 err, 0);
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002805 if (!skb)
2806 return NULL;
2807
2808 skb_reserve(skb, reserve);
2809 skb_put(skb, linear);
2810 skb->data_len = len - linear;
2811 skb->len += len - linear;
2812
2813 return skb;
2814}
2815
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01002816static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002817{
2818 struct sock *sk = sock->sk;
Steffen Hurrle342dfc32014-01-17 22:53:15 +01002819 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002820 struct sk_buff *skb;
2821 struct net_device *dev;
Al Viro0e11c912006-11-08 00:26:29 -08002822 __be16 proto;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002823 unsigned char *addr;
Ben Greear827d9782011-06-01 07:18:53 +00002824 int err, reserve = 0;
Edward Jeec7d39e32015-10-08 14:56:49 -07002825 struct sockcm_cookie sockc;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002826 struct virtio_net_hdr vnet_hdr = { 0 };
2827 int offset = 0;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002828 struct packet_sock *po = pkt_sk(sk);
Willem de Bruijn82849542017-02-07 15:57:21 -05002829 int hlen, tlen, linear;
Ben Greear3bdc0eb2012-02-11 15:39:30 +00002830 int extra_len = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002831
2832 /*
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002833 * Get and verify the address.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002834 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002835
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01002836 if (likely(saddr == NULL)) {
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002837 dev = packet_cached_dev_get(po);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002838 proto = po->num;
2839 addr = NULL;
2840 } else {
2841 err = -EINVAL;
2842 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2843 goto out;
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07002844 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2845 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002846 proto = saddr->sll_protocol;
2847 addr = saddr->sll_addr;
Ben Greear827d9782011-06-01 07:18:53 +00002848 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002849 }
2850
Linus Torvalds1da177e2005-04-16 15:20:36 -07002851 err = -ENXIO;
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002852 if (unlikely(dev == NULL))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002853 goto out_unlock;
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002854 err = -ENETDOWN;
2855 if (unlikely(!(dev->flags & IFF_UP)))
2856 goto out_unlock;
2857
Yoshihiro Shimodaedbe7742016-07-19 14:40:51 +09002858 sockc.tsflags = sk->sk_tsflags;
Edward Jeec7d39e32015-10-08 14:56:49 -07002859 sockc.mark = sk->sk_mark;
2860 if (msg->msg_controllen) {
2861 err = sock_cmsg_send(sk, msg, &sockc);
2862 if (unlikely(err))
2863 goto out_unlock;
2864 }
2865
Linus Torvalds1da177e2005-04-16 15:20:36 -07002866 if (sock->type == SOCK_RAW)
2867 reserve = dev->hard_header_len;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002868 if (po->has_vnet_hdr) {
Willem de Bruijn16cc1402016-02-03 18:02:14 -05002869 err = packet_snd_vnet_parse(msg, &len, &vnet_hdr);
2870 if (err)
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002871 goto out_unlock;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002872 }
2873
Ben Greear3bdc0eb2012-02-11 15:39:30 +00002874 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
2875 if (!netif_supports_nofcs(dev)) {
2876 err = -EPROTONOSUPPORT;
2877 goto out_unlock;
2878 }
2879 extra_len = 4; /* We're doing our own CRC */
2880 }
2881
Linus Torvalds1da177e2005-04-16 15:20:36 -07002882 err = -EMSGSIZE;
Willem de Bruijn16cc1402016-02-03 18:02:14 -05002883 if (!vnet_hdr.gso_type &&
2884 (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002885 goto out_unlock;
2886
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002887 err = -ENOBUFS;
Herbert Xuae641942011-11-18 02:20:04 +00002888 hlen = LL_RESERVED_SPACE(dev);
2889 tlen = dev->needed_tailroom;
Willem de Bruijn82849542017-02-07 15:57:21 -05002890 linear = __virtio16_to_cpu(vio_le(), vnet_hdr.hdr_len);
2891 linear = max(linear, min_t(int, len, dev->hard_header_len));
2892 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, linear,
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002893 msg->msg_flags & MSG_DONTWAIT, &err);
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002894 if (skb == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002895 goto out_unlock;
2896
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002897 skb_set_network_header(skb, reserve);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002898
Stephen Hemminger0c4e8582007-10-09 01:36:32 -07002899 err = -EINVAL;
Willem de Bruijn9c707762014-11-19 13:10:16 -05002900 if (sock->type == SOCK_DGRAM) {
2901 offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len);
Christoph Jaeger46d2cfb2015-01-11 13:01:16 -05002902 if (unlikely(offset < 0))
Willem de Bruijn9c707762014-11-19 13:10:16 -05002903 goto out_free;
Willem de Bruijn9c707762014-11-19 13:10:16 -05002904 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002905
2906 /* Returns -EFAULT on error */
Al Viroc0371da2014-11-24 10:42:55 -05002907 err = skb_copy_datagram_from_iter(skb, offset, &msg->msg_iter, len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002908 if (err)
2909 goto out_free;
Daniel Borkmannbf84a012013-04-14 08:08:13 +00002910
Willem de Bruijn9ed988c2016-03-09 21:58:34 -05002911 if (sock->type == SOCK_RAW &&
2912 !dev_validate_header(dev, skb->data, len)) {
2913 err = -EINVAL;
2914 goto out_free;
2915 }
2916
Soheil Hassas Yeganehc14ac942016-04-02 23:08:12 -04002917 sock_tx_timestamp(sk, sockc.tsflags, &skb_shinfo(skb)->tx_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002918
Willem de Bruijn16cc1402016-02-03 18:02:14 -05002919 if (!vnet_hdr.gso_type && (len > dev->mtu + reserve + extra_len) &&
Daniel Borkmann3c70c132015-11-11 23:25:42 +01002920 !packet_extra_vlan_len_allowed(dev, skb)) {
2921 err = -EMSGSIZE;
2922 goto out_free;
Ben Greear57f89bf2011-02-11 09:35:18 +00002923 }
2924
David S. Miller09effa62013-08-07 17:11:00 -07002925 skb->protocol = proto;
2926 skb->dev = dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002927 skb->priority = sk->sk_priority;
Edward Jeec7d39e32015-10-08 14:56:49 -07002928 skb->mark = sockc.mark;
Daniel Borkmann0fd5d572014-02-16 15:55:22 +01002929
2930 packet_pick_tx_queue(dev, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002931
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002932 if (po->has_vnet_hdr) {
Willem de Bruijn16cc1402016-02-03 18:02:14 -05002933 err = packet_snd_vnet_gso(skb, &vnet_hdr);
2934 if (err)
2935 goto out_free;
2936 len += sizeof(vnet_hdr);
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002937 }
2938
Daniel Borkmann8fd6c802015-11-11 23:25:41 +01002939 skb_probe_transport_header(skb, reserve);
2940
Ben Greear3bdc0eb2012-02-11 15:39:30 +00002941 if (unlikely(extra_len == 4))
2942 skb->no_fcs = 1;
2943
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01002944 err = po->xmit(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002945 if (err > 0 && (err = net_xmit_errno(err)) != 0)
2946 goto out_unlock;
2947
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002948 dev_put(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002949
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002950 return len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002951
2952out_free:
2953 kfree_skb(skb);
2954out_unlock:
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002955 if (dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002956 dev_put(dev);
2957out:
2958 return err;
2959}
2960
Ying Xue1b784142015-03-02 15:37:48 +08002961static int packet_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
Johann Baudy69e3c752009-05-18 22:11:22 -07002962{
Johann Baudy69e3c752009-05-18 22:11:22 -07002963 struct sock *sk = sock->sk;
2964 struct packet_sock *po = pkt_sk(sk);
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01002965
Johann Baudy69e3c752009-05-18 22:11:22 -07002966 if (po->tx_ring.pg_vec)
2967 return tpacket_snd(po, msg);
2968 else
Johann Baudy69e3c752009-05-18 22:11:22 -07002969 return packet_snd(sock, msg, len);
2970}
2971
Linus Torvalds1da177e2005-04-16 15:20:36 -07002972/*
2973 * Close a PACKET socket. This is fairly simple. We immediately go
2974 * to 'closed' state and remove our protocol entry in the device list.
2975 */
2976
2977static int packet_release(struct socket *sock)
2978{
2979 struct sock *sk = sock->sk;
2980 struct packet_sock *po;
Anoob Soman9206efc2017-02-15 20:25:39 +00002981 struct packet_fanout *f;
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08002982 struct net *net;
chetan lokef6fb8f102011-08-19 10:18:16 +00002983 union tpacket_req_u req_u;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002984
2985 if (!sk)
2986 return 0;
2987
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09002988 net = sock_net(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002989 po = pkt_sk(sk);
2990
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00002991 mutex_lock(&net->packet.sklist_lock);
stephen hemminger808f5112010-02-22 07:57:18 +00002992 sk_del_node_init_rcu(sk);
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00002993 mutex_unlock(&net->packet.sklist_lock);
2994
2995 preempt_disable();
Eric Dumazet920de802008-11-24 00:09:29 -08002996 sock_prot_inuse_add(net, sk->sk_prot, -1);
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00002997 preempt_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002998
stephen hemminger808f5112010-02-22 07:57:18 +00002999 spin_lock(&po->bind_lock);
David S. Millerce06b032011-07-04 01:44:29 -07003000 unregister_prot_hook(sk, false);
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01003001 packet_cached_dev_reset(po);
3002
Ben Greear160ff182011-06-01 07:18:52 +00003003 if (po->prot_hook.dev) {
3004 dev_put(po->prot_hook.dev);
3005 po->prot_hook.dev = NULL;
3006 }
stephen hemminger808f5112010-02-22 07:57:18 +00003007 spin_unlock(&po->bind_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003008
Linus Torvalds1da177e2005-04-16 15:20:36 -07003009 packet_flush_mclist(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003010
Phil Sutter9665d5d2013-02-01 07:21:41 +00003011 if (po->rx_ring.pg_vec) {
3012 memset(&req_u, 0, sizeof(req_u));
chetan lokef6fb8f102011-08-19 10:18:16 +00003013 packet_set_ring(sk, &req_u, 1, 0);
Phil Sutter9665d5d2013-02-01 07:21:41 +00003014 }
Johann Baudy69e3c752009-05-18 22:11:22 -07003015
Phil Sutter9665d5d2013-02-01 07:21:41 +00003016 if (po->tx_ring.pg_vec) {
3017 memset(&req_u, 0, sizeof(req_u));
chetan lokef6fb8f102011-08-19 10:18:16 +00003018 packet_set_ring(sk, &req_u, 1, 1);
Phil Sutter9665d5d2013-02-01 07:21:41 +00003019 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003020
Anoob Soman9206efc2017-02-15 20:25:39 +00003021 f = fanout_release(sk);
David S. Millerdc99f602011-07-05 01:45:05 -07003022
stephen hemminger808f5112010-02-22 07:57:18 +00003023 synchronize_net();
Anoob Soman9206efc2017-02-15 20:25:39 +00003024
3025 if (f) {
3026 fanout_release_data(f);
3027 kfree(f);
3028 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003029 /*
3030 * Now the socket is dead. No more input will appear.
3031 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07003032 sock_orphan(sk);
3033 sock->sk = NULL;
3034
3035 /* Purge queues */
3036
3037 skb_queue_purge(&sk->sk_receive_queue);
Daniel Borkmannb0138402014-01-15 16:25:36 +01003038 packet_free_pending(po);
Pavel Emelyanov17ab56a2007-11-10 21:38:48 -08003039 sk_refcnt_debug_release(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003040
3041 sock_put(sk);
3042 return 0;
3043}
3044
3045/*
3046 * Attach a packet hook.
3047 */
3048
Francesco Ruggeri30f7ea12015-11-05 08:16:14 -08003049static int packet_do_bind(struct sock *sk, const char *name, int ifindex,
3050 __be16 proto)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003051{
3052 struct packet_sock *po = pkt_sk(sk);
Lars Westerhoff158cd4a2015-07-28 01:32:21 +03003053 struct net_device *dev_curr;
Daniel Borkmann902fefb2014-01-15 16:25:34 +01003054 __be16 proto_curr;
3055 bool need_rehook;
Francesco Ruggeri30f7ea12015-11-05 08:16:14 -08003056 struct net_device *dev = NULL;
3057 int ret = 0;
3058 bool unlisted = false;
David S. Millerdc99f602011-07-05 01:45:05 -07003059
Francesco Ruggeri30f7ea12015-11-05 08:16:14 -08003060 if (po->fanout)
David S. Millerdc99f602011-07-05 01:45:05 -07003061 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003062
3063 lock_sock(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003064 spin_lock(&po->bind_lock);
Francesco Ruggeri30f7ea12015-11-05 08:16:14 -08003065 rcu_read_lock();
3066
3067 if (name) {
3068 dev = dev_get_by_name_rcu(sock_net(sk), name);
3069 if (!dev) {
3070 ret = -ENODEV;
3071 goto out_unlock;
3072 }
3073 } else if (ifindex) {
3074 dev = dev_get_by_index_rcu(sock_net(sk), ifindex);
3075 if (!dev) {
3076 ret = -ENODEV;
3077 goto out_unlock;
3078 }
3079 }
3080
3081 if (dev)
3082 dev_hold(dev);
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01003083
Daniel Borkmann902fefb2014-01-15 16:25:34 +01003084 proto_curr = po->prot_hook.type;
3085 dev_curr = po->prot_hook.dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003086
Daniel Borkmann902fefb2014-01-15 16:25:34 +01003087 need_rehook = proto_curr != proto || dev_curr != dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003088
Daniel Borkmann902fefb2014-01-15 16:25:34 +01003089 if (need_rehook) {
Francesco Ruggeri30f7ea12015-11-05 08:16:14 -08003090 if (po->running) {
3091 rcu_read_unlock();
3092 __unregister_prot_hook(sk, true);
3093 rcu_read_lock();
3094 dev_curr = po->prot_hook.dev;
3095 if (dev)
3096 unlisted = !dev_get_by_index_rcu(sock_net(sk),
3097 dev->ifindex);
3098 }
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01003099
Daniel Borkmann902fefb2014-01-15 16:25:34 +01003100 po->num = proto;
3101 po->prot_hook.type = proto;
Daniel Borkmann902fefb2014-01-15 16:25:34 +01003102
Francesco Ruggeri30f7ea12015-11-05 08:16:14 -08003103 if (unlikely(unlisted)) {
3104 dev_put(dev);
3105 po->prot_hook.dev = NULL;
3106 po->ifindex = -1;
3107 packet_cached_dev_reset(po);
3108 } else {
3109 po->prot_hook.dev = dev;
3110 po->ifindex = dev ? dev->ifindex : 0;
3111 packet_cached_dev_assign(po, dev);
3112 }
Daniel Borkmann902fefb2014-01-15 16:25:34 +01003113 }
Lars Westerhoff158cd4a2015-07-28 01:32:21 +03003114 if (dev_curr)
3115 dev_put(dev_curr);
Daniel Borkmann902fefb2014-01-15 16:25:34 +01003116
3117 if (proto == 0 || !need_rehook)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003118 goto out_unlock;
3119
Francesco Ruggeri30f7ea12015-11-05 08:16:14 -08003120 if (!unlisted && (!dev || (dev->flags & IFF_UP))) {
David S. Millerce06b032011-07-04 01:44:29 -07003121 register_prot_hook(sk);
Urs Thuermannbe85d4a2007-11-12 21:05:20 -08003122 } else {
3123 sk->sk_err = ENETDOWN;
3124 if (!sock_flag(sk, SOCK_DEAD))
3125 sk->sk_error_report(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003126 }
3127
3128out_unlock:
Francesco Ruggeri30f7ea12015-11-05 08:16:14 -08003129 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003130 spin_unlock(&po->bind_lock);
3131 release_sock(sk);
Francesco Ruggeri30f7ea12015-11-05 08:16:14 -08003132 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003133}
3134
3135/*
3136 * Bind a packet socket to a device
3137 */
3138
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003139static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
3140 int addr_len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003141{
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003142 struct sock *sk = sock->sk;
Alexander Potapenko62fe0522017-03-01 12:57:20 +01003143 char name[sizeof(uaddr->sa_data) + 1];
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003144
Linus Torvalds1da177e2005-04-16 15:20:36 -07003145 /*
3146 * Check legality
3147 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003148
Kris Katterjohn8ae55f02006-01-23 16:28:02 -08003149 if (addr_len != sizeof(struct sockaddr))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003150 return -EINVAL;
Alexander Potapenko62fe0522017-03-01 12:57:20 +01003151 /* uaddr->sa_data comes from the userspace, it's not guaranteed to be
3152 * zero-terminated.
3153 */
3154 memcpy(name, uaddr->sa_data, sizeof(uaddr->sa_data));
3155 name[sizeof(uaddr->sa_data)] = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003156
Francesco Ruggeri30f7ea12015-11-05 08:16:14 -08003157 return packet_do_bind(sk, name, 0, pkt_sk(sk)->num);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003158}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003159
3160static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
3161{
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003162 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
3163 struct sock *sk = sock->sk;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003164
3165 /*
3166 * Check legality
3167 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003168
Linus Torvalds1da177e2005-04-16 15:20:36 -07003169 if (addr_len < sizeof(struct sockaddr_ll))
3170 return -EINVAL;
3171 if (sll->sll_family != AF_PACKET)
3172 return -EINVAL;
3173
Francesco Ruggeri30f7ea12015-11-05 08:16:14 -08003174 return packet_do_bind(sk, NULL, sll->sll_ifindex,
3175 sll->sll_protocol ? : pkt_sk(sk)->num);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003176}
3177
3178static struct proto packet_proto = {
3179 .name = "PACKET",
3180 .owner = THIS_MODULE,
3181 .obj_size = sizeof(struct packet_sock),
3182};
3183
3184/*
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003185 * Create a packet of type SOCK_PACKET.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003186 */
3187
Eric Paris3f378b62009-11-05 22:18:14 -08003188static int packet_create(struct net *net, struct socket *sock, int protocol,
3189 int kern)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003190{
3191 struct sock *sk;
3192 struct packet_sock *po;
Al Viro0e11c912006-11-08 00:26:29 -08003193 __be16 proto = (__force __be16)protocol; /* weird, but documented */
Linus Torvalds1da177e2005-04-16 15:20:36 -07003194 int err;
3195
Eric W. Biedermandf008c92012-11-16 03:03:07 +00003196 if (!ns_capable(net->user_ns, CAP_NET_RAW))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003197 return -EPERM;
David S. Millerbe020972007-05-29 13:16:31 -07003198 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
3199 sock->type != SOCK_PACKET)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003200 return -ESOCKTNOSUPPORT;
3201
3202 sock->state = SS_UNCONNECTED;
3203
3204 err = -ENOBUFS;
Eric W. Biederman11aa9c22015-05-08 21:09:13 -05003205 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto, kern);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003206 if (sk == NULL)
3207 goto out;
3208
3209 sock->ops = &packet_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003210 if (sock->type == SOCK_PACKET)
3211 sock->ops = &packet_ops_spkt;
David S. Millerbe020972007-05-29 13:16:31 -07003212
Linus Torvalds1da177e2005-04-16 15:20:36 -07003213 sock_init_data(sock, sk);
3214
3215 po = pkt_sk(sk);
3216 sk->sk_family = PF_PACKET;
Al Viro0e11c912006-11-08 00:26:29 -08003217 po->num = proto;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01003218 po->xmit = dev_queue_xmit;
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01003219
Daniel Borkmannb0138402014-01-15 16:25:36 +01003220 err = packet_alloc_pending(po);
3221 if (err)
3222 goto out2;
3223
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01003224 packet_cached_dev_reset(po);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003225
3226 sk->sk_destruct = packet_sock_destruct;
Pavel Emelyanov17ab56a2007-11-10 21:38:48 -08003227 sk_refcnt_debug_inc(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003228
3229 /*
3230 * Attach a protocol block
3231 */
3232
3233 spin_lock_init(&po->bind_lock);
Herbert Xu905db442009-01-30 14:12:06 -08003234 mutex_init(&po->pg_vec_lock);
Willem de Bruijn0648ab72015-05-12 11:56:46 -04003235 po->rollover = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003236 po->prot_hook.func = packet_rcv;
David S. Millerbe020972007-05-29 13:16:31 -07003237
Linus Torvalds1da177e2005-04-16 15:20:36 -07003238 if (sock->type == SOCK_PACKET)
3239 po->prot_hook.func = packet_rcv_spkt;
David S. Millerbe020972007-05-29 13:16:31 -07003240
Linus Torvalds1da177e2005-04-16 15:20:36 -07003241 po->prot_hook.af_packet_priv = sk;
3242
Al Viro0e11c912006-11-08 00:26:29 -08003243 if (proto) {
3244 po->prot_hook.type = proto;
David S. Millerce06b032011-07-04 01:44:29 -07003245 register_prot_hook(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003246 }
3247
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00003248 mutex_lock(&net->packet.sklist_lock);
stephen hemminger808f5112010-02-22 07:57:18 +00003249 sk_add_node_rcu(sk, &net->packet.sklist);
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00003250 mutex_unlock(&net->packet.sklist_lock);
3251
3252 preempt_disable();
Eric Dumazet36804532008-11-19 14:25:35 -08003253 sock_prot_inuse_add(net, &packet_proto, 1);
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00003254 preempt_enable();
stephen hemminger808f5112010-02-22 07:57:18 +00003255
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003256 return 0;
Daniel Borkmannb0138402014-01-15 16:25:36 +01003257out2:
3258 sk_free(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003259out:
3260 return err;
3261}
3262
3263/*
3264 * Pull a packet from our receive queue and hand it to the user.
3265 * If necessary we block.
3266 */
3267
Ying Xue1b784142015-03-02 15:37:48 +08003268static int packet_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
3269 int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003270{
3271 struct sock *sk = sock->sk;
3272 struct sk_buff *skb;
3273 int copied, err;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003274 int vnet_hdr_len = 0;
Eyal Birger2472d762015-03-01 14:58:28 +02003275 unsigned int origlen = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003276
3277 err = -EINVAL;
Richard Cochraned85b562010-04-07 22:41:28 +00003278 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003279 goto out;
3280
3281#if 0
3282 /* What error should we return now? EUNATTACH? */
3283 if (pkt_sk(sk)->ifindex < 0)
3284 return -ENODEV;
3285#endif
3286
Richard Cochraned85b562010-04-07 22:41:28 +00003287 if (flags & MSG_ERRQUEUE) {
Richard Cochrancb820f82013-07-19 19:40:09 +02003288 err = sock_recv_errqueue(sk, msg, len,
3289 SOL_PACKET, PACKET_TX_TIMESTAMP);
Richard Cochraned85b562010-04-07 22:41:28 +00003290 goto out;
3291 }
3292
Linus Torvalds1da177e2005-04-16 15:20:36 -07003293 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -07003294 * Call the generic datagram receiver. This handles all sorts
3295 * of horrible races and re-entrancy so we can forget about it
3296 * in the protocol layers.
3297 *
3298 * Now it will return ENETDOWN, if device have just gone down,
3299 * but then it will block.
3300 */
3301
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003302 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003303
3304 /*
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003305 * An error occurred so return it. Because skb_recv_datagram()
Linus Torvalds1da177e2005-04-16 15:20:36 -07003306 * handles the blocking we don't see and worry about blocking
3307 * retries.
3308 */
3309
Kris Katterjohn8ae55f02006-01-23 16:28:02 -08003310 if (skb == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003311 goto out;
3312
Willem de Bruijn2ccdbaa2015-05-12 11:56:48 -04003313 if (pkt_sk(sk)->pressure)
3314 packet_rcv_has_room(pkt_sk(sk), NULL);
3315
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003316 if (pkt_sk(sk)->has_vnet_hdr) {
Willem de Bruijn16cc1402016-02-03 18:02:14 -05003317 err = packet_rcv_vnet(msg, skb, &len);
3318 if (err)
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003319 goto out_free;
Willem de Bruijn16cc1402016-02-03 18:02:14 -05003320 vnet_hdr_len = sizeof(struct virtio_net_hdr);
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003321 }
3322
Hannes Frederic Sowaf3d33422013-11-21 03:14:22 +01003323 /* You lose any data beyond the buffer you gave. If it worries
3324 * a user program they can ask the device for its MTU
3325 * anyway.
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07003326 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07003327 copied = skb->len;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003328 if (copied > len) {
3329 copied = len;
3330 msg->msg_flags |= MSG_TRUNC;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003331 }
3332
David S. Miller51f3d022014-11-05 16:46:40 -05003333 err = skb_copy_datagram_msg(skb, 0, msg, copied);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003334 if (err)
3335 goto out_free;
3336
Eyal Birger2472d762015-03-01 14:58:28 +02003337 if (sock->type != SOCK_PACKET) {
3338 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
3339
3340 /* Original length was stored in sockaddr_ll fields */
3341 origlen = PACKET_SKB_CB(skb)->sa.origlen;
3342 sll->sll_family = AF_PACKET;
3343 sll->sll_protocol = skb->protocol;
3344 }
3345
Neil Horman3b885782009-10-12 13:26:31 -07003346 sock_recv_ts_and_drops(msg, sk, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003347
Hannes Frederic Sowaf3d33422013-11-21 03:14:22 +01003348 if (msg->msg_name) {
3349 /* If the address length field is there to be filled
3350 * in, we fill it in now.
3351 */
3352 if (sock->type == SOCK_PACKET) {
Steffen Hurrle342dfc32014-01-17 22:53:15 +01003353 __sockaddr_check_size(sizeof(struct sockaddr_pkt));
Hannes Frederic Sowaf3d33422013-11-21 03:14:22 +01003354 msg->msg_namelen = sizeof(struct sockaddr_pkt);
3355 } else {
3356 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
Eyal Birger2472d762015-03-01 14:58:28 +02003357
Hannes Frederic Sowaf3d33422013-11-21 03:14:22 +01003358 msg->msg_namelen = sll->sll_halen +
3359 offsetof(struct sockaddr_ll, sll_addr);
3360 }
Herbert Xuffbc6112007-02-04 23:33:10 -08003361 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
3362 msg->msg_namelen);
Hannes Frederic Sowaf3d33422013-11-21 03:14:22 +01003363 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003364
Herbert Xu8dc41942007-02-04 23:31:32 -08003365 if (pkt_sk(sk)->auxdata) {
Herbert Xuffbc6112007-02-04 23:33:10 -08003366 struct tpacket_auxdata aux;
3367
3368 aux.tp_status = TP_STATUS_USER;
3369 if (skb->ip_summed == CHECKSUM_PARTIAL)
3370 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
Alexander Drozdov682f0482015-03-23 09:11:13 +03003371 else if (skb->pkt_type != PACKET_OUTGOING &&
3372 (skb->ip_summed == CHECKSUM_COMPLETE ||
3373 skb_csum_unnecessary(skb)))
3374 aux.tp_status |= TP_STATUS_CSUM_VALID;
3375
Eyal Birger2472d762015-03-01 14:58:28 +02003376 aux.tp_len = origlen;
Herbert Xuffbc6112007-02-04 23:33:10 -08003377 aux.tp_snaplen = skb->len;
3378 aux.tp_mac = 0;
Arnaldo Carvalho de Melobbe735e2007-03-10 22:16:10 -03003379 aux.tp_net = skb_network_offset(skb);
Jiri Pirkodf8a39d2015-01-13 17:13:44 +01003380 if (skb_vlan_tag_present(skb)) {
3381 aux.tp_vlan_tci = skb_vlan_tag_get(skb);
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +09003382 aux.tp_vlan_tpid = ntohs(skb->vlan_proto);
3383 aux.tp_status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
Ben Greeara3bcc232011-06-01 06:49:10 +00003384 } else {
3385 aux.tp_vlan_tci = 0;
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +09003386 aux.tp_vlan_tpid = 0;
Ben Greeara3bcc232011-06-01 06:49:10 +00003387 }
Herbert Xuffbc6112007-02-04 23:33:10 -08003388 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
Herbert Xu8dc41942007-02-04 23:31:32 -08003389 }
3390
Linus Torvalds1da177e2005-04-16 15:20:36 -07003391 /*
3392 * Free or return the buffer as appropriate. Again this
3393 * hides all the races and re-entrancy issues from us.
3394 */
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003395 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003396
3397out_free:
3398 skb_free_datagram(sk, skb);
3399out:
3400 return err;
3401}
3402
Linus Torvalds1da177e2005-04-16 15:20:36 -07003403static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
3404 int *uaddr_len, int peer)
3405{
3406 struct net_device *dev;
3407 struct sock *sk = sock->sk;
3408
3409 if (peer)
3410 return -EOPNOTSUPP;
3411
3412 uaddr->sa_family = AF_PACKET;
Daniel Borkmann2dc85bf2013-06-12 16:02:27 +02003413 memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data));
Eric Dumazet654d1f82009-11-02 10:43:32 +01003414 rcu_read_lock();
3415 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
3416 if (dev)
Daniel Borkmann2dc85bf2013-06-12 16:02:27 +02003417 strlcpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data));
Eric Dumazet654d1f82009-11-02 10:43:32 +01003418 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003419 *uaddr_len = sizeof(*uaddr);
3420
3421 return 0;
3422}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003423
3424static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
3425 int *uaddr_len, int peer)
3426{
3427 struct net_device *dev;
3428 struct sock *sk = sock->sk;
3429 struct packet_sock *po = pkt_sk(sk);
Cyrill Gorcunov13cfa972009-11-08 05:51:19 +00003430 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003431
3432 if (peer)
3433 return -EOPNOTSUPP;
3434
3435 sll->sll_family = AF_PACKET;
3436 sll->sll_ifindex = po->ifindex;
3437 sll->sll_protocol = po->num;
Vasiliy Kulikov67286642010-11-10 12:09:10 -08003438 sll->sll_pkttype = 0;
Eric Dumazet654d1f82009-11-02 10:43:32 +01003439 rcu_read_lock();
3440 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003441 if (dev) {
3442 sll->sll_hatype = dev->type;
3443 sll->sll_halen = dev->addr_len;
3444 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003445 } else {
3446 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
3447 sll->sll_halen = 0;
3448 }
Eric Dumazet654d1f82009-11-02 10:43:32 +01003449 rcu_read_unlock();
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07003450 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003451
3452 return 0;
3453}
3454
Wang Chen2aeb0b82008-07-14 20:49:46 -07003455static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
3456 int what)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003457{
3458 switch (i->type) {
3459 case PACKET_MR_MULTICAST:
Jiri Pirko11625632010-03-02 20:40:01 +00003460 if (i->alen != dev->addr_len)
3461 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003462 if (what > 0)
Jiri Pirko22bedad32010-04-01 21:22:57 +00003463 return dev_mc_add(dev, i->addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003464 else
Jiri Pirko22bedad32010-04-01 21:22:57 +00003465 return dev_mc_del(dev, i->addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003466 break;
3467 case PACKET_MR_PROMISC:
Wang Chen2aeb0b82008-07-14 20:49:46 -07003468 return dev_set_promiscuity(dev, what);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003469 case PACKET_MR_ALLMULTI:
Wang Chen2aeb0b82008-07-14 20:49:46 -07003470 return dev_set_allmulti(dev, what);
Eric W. Biedermand95ed922009-05-19 18:27:17 +00003471 case PACKET_MR_UNICAST:
Jiri Pirko11625632010-03-02 20:40:01 +00003472 if (i->alen != dev->addr_len)
3473 return -EINVAL;
Eric W. Biedermand95ed922009-05-19 18:27:17 +00003474 if (what > 0)
Jiri Pirkoa748ee22010-04-01 21:22:09 +00003475 return dev_uc_add(dev, i->addr);
Eric W. Biedermand95ed922009-05-19 18:27:17 +00003476 else
Jiri Pirkoa748ee22010-04-01 21:22:09 +00003477 return dev_uc_del(dev, i->addr);
Eric W. Biedermand95ed922009-05-19 18:27:17 +00003478 break;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003479 default:
3480 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003481 }
Wang Chen2aeb0b82008-07-14 20:49:46 -07003482 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003483}
3484
Francesco Ruggeri82f17092015-03-09 11:51:04 -07003485static void packet_dev_mclist_delete(struct net_device *dev,
3486 struct packet_mclist **mlp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003487{
Francesco Ruggeri82f17092015-03-09 11:51:04 -07003488 struct packet_mclist *ml;
3489
3490 while ((ml = *mlp) != NULL) {
3491 if (ml->ifindex == dev->ifindex) {
3492 packet_dev_mc(dev, ml, -1);
3493 *mlp = ml->next;
3494 kfree(ml);
3495 } else
3496 mlp = &ml->next;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003497 }
3498}
3499
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07003500static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003501{
3502 struct packet_sock *po = pkt_sk(sk);
3503 struct packet_mclist *ml, *i;
3504 struct net_device *dev;
3505 int err;
3506
3507 rtnl_lock();
3508
3509 err = -ENODEV;
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09003510 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003511 if (!dev)
3512 goto done;
3513
3514 err = -EINVAL;
Jiri Pirko11625632010-03-02 20:40:01 +00003515 if (mreq->mr_alen > dev->addr_len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003516 goto done;
3517
3518 err = -ENOBUFS;
Kris Katterjohn8b3a7002006-01-11 15:56:43 -08003519 i = kmalloc(sizeof(*i), GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003520 if (i == NULL)
3521 goto done;
3522
3523 err = 0;
3524 for (ml = po->mclist; ml; ml = ml->next) {
3525 if (ml->ifindex == mreq->mr_ifindex &&
3526 ml->type == mreq->mr_type &&
3527 ml->alen == mreq->mr_alen &&
3528 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3529 ml->count++;
3530 /* Free the new element ... */
3531 kfree(i);
3532 goto done;
3533 }
3534 }
3535
3536 i->type = mreq->mr_type;
3537 i->ifindex = mreq->mr_ifindex;
3538 i->alen = mreq->mr_alen;
3539 memcpy(i->addr, mreq->mr_address, i->alen);
Mathias Krause309cf372016-04-10 12:52:28 +02003540 memset(i->addr + i->alen, 0, sizeof(i->addr) - i->alen);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003541 i->count = 1;
3542 i->next = po->mclist;
3543 po->mclist = i;
Wang Chen2aeb0b82008-07-14 20:49:46 -07003544 err = packet_dev_mc(dev, i, 1);
3545 if (err) {
3546 po->mclist = i->next;
3547 kfree(i);
3548 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003549
3550done:
3551 rtnl_unlock();
3552 return err;
3553}
3554
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07003555static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003556{
3557 struct packet_mclist *ml, **mlp;
3558
3559 rtnl_lock();
3560
3561 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
3562 if (ml->ifindex == mreq->mr_ifindex &&
3563 ml->type == mreq->mr_type &&
3564 ml->alen == mreq->mr_alen &&
3565 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3566 if (--ml->count == 0) {
3567 struct net_device *dev;
3568 *mlp = ml->next;
Eric Dumazetad959e72009-10-16 06:38:46 +00003569 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3570 if (dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003571 packet_dev_mc(dev, ml, -1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003572 kfree(ml);
3573 }
Francesco Ruggeri82f17092015-03-09 11:51:04 -07003574 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003575 }
3576 }
3577 rtnl_unlock();
Francesco Ruggeri82f17092015-03-09 11:51:04 -07003578 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003579}
3580
3581static void packet_flush_mclist(struct sock *sk)
3582{
3583 struct packet_sock *po = pkt_sk(sk);
3584 struct packet_mclist *ml;
3585
3586 if (!po->mclist)
3587 return;
3588
3589 rtnl_lock();
3590 while ((ml = po->mclist) != NULL) {
3591 struct net_device *dev;
3592
3593 po->mclist = ml->next;
Eric Dumazetad959e72009-10-16 06:38:46 +00003594 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3595 if (dev != NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003596 packet_dev_mc(dev, ml, -1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003597 kfree(ml);
3598 }
3599 rtnl_unlock();
3600}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003601
3602static int
David S. Millerb7058842009-09-30 16:12:20 -07003603packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003604{
3605 struct sock *sk = sock->sk;
Herbert Xu8dc41942007-02-04 23:31:32 -08003606 struct packet_sock *po = pkt_sk(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003607 int ret;
3608
3609 if (level != SOL_PACKET)
3610 return -ENOPROTOOPT;
3611
Johann Baudy69e3c752009-05-18 22:11:22 -07003612 switch (optname) {
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003613 case PACKET_ADD_MEMBERSHIP:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003614 case PACKET_DROP_MEMBERSHIP:
3615 {
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07003616 struct packet_mreq_max mreq;
3617 int len = optlen;
3618 memset(&mreq, 0, sizeof(mreq));
3619 if (len < sizeof(struct packet_mreq))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003620 return -EINVAL;
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07003621 if (len > sizeof(mreq))
3622 len = sizeof(mreq);
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003623 if (copy_from_user(&mreq, optval, len))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003624 return -EFAULT;
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07003625 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3626 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003627 if (optname == PACKET_ADD_MEMBERSHIP)
3628 ret = packet_mc_add(sk, &mreq);
3629 else
3630 ret = packet_mc_drop(sk, &mreq);
3631 return ret;
3632 }
David S. Millera2efcfa2007-05-29 13:12:50 -07003633
Linus Torvalds1da177e2005-04-16 15:20:36 -07003634 case PACKET_RX_RING:
Johann Baudy69e3c752009-05-18 22:11:22 -07003635 case PACKET_TX_RING:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003636 {
chetan lokef6fb8f102011-08-19 10:18:16 +00003637 union tpacket_req_u req_u;
3638 int len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003639
chetan lokef6fb8f102011-08-19 10:18:16 +00003640 switch (po->tp_version) {
3641 case TPACKET_V1:
3642 case TPACKET_V2:
3643 len = sizeof(req_u.req);
3644 break;
3645 case TPACKET_V3:
3646 default:
3647 len = sizeof(req_u.req3);
3648 break;
3649 }
3650 if (optlen < len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003651 return -EINVAL;
chetan lokef6fb8f102011-08-19 10:18:16 +00003652 if (copy_from_user(&req_u.req, optval, len))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003653 return -EFAULT;
chetan lokef6fb8f102011-08-19 10:18:16 +00003654 return packet_set_ring(sk, &req_u, 0,
3655 optname == PACKET_TX_RING);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003656 }
3657 case PACKET_COPY_THRESH:
3658 {
3659 int val;
3660
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003661 if (optlen != sizeof(val))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003662 return -EINVAL;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003663 if (copy_from_user(&val, optval, sizeof(val)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003664 return -EFAULT;
3665
3666 pkt_sk(sk)->copy_thresh = val;
3667 return 0;
3668 }
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003669 case PACKET_VERSION:
3670 {
3671 int val;
3672
3673 if (optlen != sizeof(val))
3674 return -EINVAL;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003675 if (copy_from_user(&val, optval, sizeof(val)))
3676 return -EFAULT;
3677 switch (val) {
3678 case TPACKET_V1:
3679 case TPACKET_V2:
chetan lokef6fb8f102011-08-19 10:18:16 +00003680 case TPACKET_V3:
Philip Pettersson84ac7262016-11-30 14:55:36 -08003681 break;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003682 default:
3683 return -EINVAL;
3684 }
Philip Pettersson84ac7262016-11-30 14:55:36 -08003685 lock_sock(sk);
3686 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
3687 ret = -EBUSY;
3688 } else {
3689 po->tp_version = val;
3690 ret = 0;
3691 }
3692 release_sock(sk);
3693 return ret;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003694 }
Patrick McHardy8913336a2008-07-18 18:05:19 -07003695 case PACKET_RESERVE:
3696 {
3697 unsigned int val;
3698
3699 if (optlen != sizeof(val))
3700 return -EINVAL;
Johann Baudy69e3c752009-05-18 22:11:22 -07003701 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
Patrick McHardy8913336a2008-07-18 18:05:19 -07003702 return -EBUSY;
3703 if (copy_from_user(&val, optval, sizeof(val)))
3704 return -EFAULT;
Andrey Konovalov1f49c8c2017-03-29 16:11:22 +02003705 if (val > INT_MAX)
3706 return -EINVAL;
Patrick McHardy8913336a2008-07-18 18:05:19 -07003707 po->tp_reserve = val;
3708 return 0;
3709 }
Johann Baudy69e3c752009-05-18 22:11:22 -07003710 case PACKET_LOSS:
3711 {
3712 unsigned int val;
3713
3714 if (optlen != sizeof(val))
3715 return -EINVAL;
3716 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3717 return -EBUSY;
3718 if (copy_from_user(&val, optval, sizeof(val)))
3719 return -EFAULT;
3720 po->tp_loss = !!val;
3721 return 0;
3722 }
Herbert Xu8dc41942007-02-04 23:31:32 -08003723 case PACKET_AUXDATA:
3724 {
3725 int val;
3726
3727 if (optlen < sizeof(val))
3728 return -EINVAL;
3729 if (copy_from_user(&val, optval, sizeof(val)))
3730 return -EFAULT;
3731
3732 po->auxdata = !!val;
3733 return 0;
3734 }
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07003735 case PACKET_ORIGDEV:
3736 {
3737 int val;
3738
3739 if (optlen < sizeof(val))
3740 return -EINVAL;
3741 if (copy_from_user(&val, optval, sizeof(val)))
3742 return -EFAULT;
3743
3744 po->origdev = !!val;
3745 return 0;
3746 }
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003747 case PACKET_VNET_HDR:
3748 {
3749 int val;
3750
3751 if (sock->type != SOCK_RAW)
3752 return -EINVAL;
3753 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3754 return -EBUSY;
3755 if (optlen < sizeof(val))
3756 return -EINVAL;
3757 if (copy_from_user(&val, optval, sizeof(val)))
3758 return -EFAULT;
3759
3760 po->has_vnet_hdr = !!val;
3761 return 0;
3762 }
Scott McMillan614f60f2010-06-02 05:53:56 -07003763 case PACKET_TIMESTAMP:
3764 {
3765 int val;
3766
3767 if (optlen != sizeof(val))
3768 return -EINVAL;
3769 if (copy_from_user(&val, optval, sizeof(val)))
3770 return -EFAULT;
3771
3772 po->tp_tstamp = val;
3773 return 0;
3774 }
David S. Millerdc99f602011-07-05 01:45:05 -07003775 case PACKET_FANOUT:
3776 {
3777 int val;
3778
3779 if (optlen != sizeof(val))
3780 return -EINVAL;
3781 if (copy_from_user(&val, optval, sizeof(val)))
3782 return -EFAULT;
3783
3784 return fanout_add(sk, val & 0xffff, val >> 16);
3785 }
Willem de Bruijn47dceb82015-08-14 22:31:34 -04003786 case PACKET_FANOUT_DATA:
3787 {
3788 if (!po->fanout)
3789 return -EINVAL;
3790
3791 return fanout_set_data(po, optval, optlen);
3792 }
Paul Chavent5920cd3a2012-11-06 23:10:47 +00003793 case PACKET_TX_HAS_OFF:
3794 {
3795 unsigned int val;
3796
3797 if (optlen != sizeof(val))
3798 return -EINVAL;
3799 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3800 return -EBUSY;
3801 if (copy_from_user(&val, optval, sizeof(val)))
3802 return -EFAULT;
3803 po->tp_tx_has_off = !!val;
3804 return 0;
3805 }
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01003806 case PACKET_QDISC_BYPASS:
3807 {
3808 int val;
3809
3810 if (optlen != sizeof(val))
3811 return -EINVAL;
3812 if (copy_from_user(&val, optval, sizeof(val)))
3813 return -EFAULT;
3814
3815 po->xmit = val ? packet_direct_xmit : dev_queue_xmit;
3816 return 0;
3817 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003818 default:
3819 return -ENOPROTOOPT;
3820 }
3821}
3822
3823static int packet_getsockopt(struct socket *sock, int level, int optname,
3824 char __user *optval, int __user *optlen)
3825{
3826 int len;
Eric Dumazetc06fff62012-04-19 21:56:11 +00003827 int val, lv = sizeof(val);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003828 struct sock *sk = sock->sk;
3829 struct packet_sock *po = pkt_sk(sk);
Eric Dumazetc06fff62012-04-19 21:56:11 +00003830 void *data = &val;
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00003831 union tpacket_stats_u st;
Willem de Bruijna9b63912015-05-12 11:56:50 -04003832 struct tpacket_rollover_stats rstats;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003833
3834 if (level != SOL_PACKET)
3835 return -ENOPROTOOPT;
3836
Kris Katterjohn8ae55f02006-01-23 16:28:02 -08003837 if (get_user(len, optlen))
3838 return -EFAULT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003839
3840 if (len < 0)
3841 return -EINVAL;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003842
Johann Baudy69e3c752009-05-18 22:11:22 -07003843 switch (optname) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003844 case PACKET_STATISTICS:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003845 spin_lock_bh(&sk->sk_receive_queue.lock);
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00003846 memcpy(&st, &po->stats, sizeof(st));
3847 memset(&po->stats, 0, sizeof(po->stats));
3848 spin_unlock_bh(&sk->sk_receive_queue.lock);
3849
chetan lokef6fb8f102011-08-19 10:18:16 +00003850 if (po->tp_version == TPACKET_V3) {
Eric Dumazetc06fff62012-04-19 21:56:11 +00003851 lv = sizeof(struct tpacket_stats_v3);
Willem de Bruijn8bcdeaf2013-08-19 16:40:22 -04003852 st.stats3.tp_packets += st.stats3.tp_drops;
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00003853 data = &st.stats3;
chetan lokef6fb8f102011-08-19 10:18:16 +00003854 } else {
Eric Dumazetc06fff62012-04-19 21:56:11 +00003855 lv = sizeof(struct tpacket_stats);
Willem de Bruijn8bcdeaf2013-08-19 16:40:22 -04003856 st.stats1.tp_packets += st.stats1.tp_drops;
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00003857 data = &st.stats1;
chetan lokef6fb8f102011-08-19 10:18:16 +00003858 }
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00003859
Linus Torvalds1da177e2005-04-16 15:20:36 -07003860 break;
Herbert Xu8dc41942007-02-04 23:31:32 -08003861 case PACKET_AUXDATA:
Herbert Xu8dc41942007-02-04 23:31:32 -08003862 val = po->auxdata;
Herbert Xu8dc41942007-02-04 23:31:32 -08003863 break;
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07003864 case PACKET_ORIGDEV:
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07003865 val = po->origdev;
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07003866 break;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003867 case PACKET_VNET_HDR:
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003868 val = po->has_vnet_hdr;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003869 break;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003870 case PACKET_VERSION:
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003871 val = po->tp_version;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003872 break;
3873 case PACKET_HDRLEN:
3874 if (len > sizeof(int))
3875 len = sizeof(int);
3876 if (copy_from_user(&val, optval, len))
3877 return -EFAULT;
3878 switch (val) {
3879 case TPACKET_V1:
3880 val = sizeof(struct tpacket_hdr);
3881 break;
3882 case TPACKET_V2:
3883 val = sizeof(struct tpacket2_hdr);
3884 break;
chetan lokef6fb8f102011-08-19 10:18:16 +00003885 case TPACKET_V3:
3886 val = sizeof(struct tpacket3_hdr);
3887 break;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003888 default:
3889 return -EINVAL;
3890 }
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003891 break;
Patrick McHardy8913336a2008-07-18 18:05:19 -07003892 case PACKET_RESERVE:
Patrick McHardy8913336a2008-07-18 18:05:19 -07003893 val = po->tp_reserve;
Patrick McHardy8913336a2008-07-18 18:05:19 -07003894 break;
Johann Baudy69e3c752009-05-18 22:11:22 -07003895 case PACKET_LOSS:
Johann Baudy69e3c752009-05-18 22:11:22 -07003896 val = po->tp_loss;
Johann Baudy69e3c752009-05-18 22:11:22 -07003897 break;
Scott McMillan614f60f2010-06-02 05:53:56 -07003898 case PACKET_TIMESTAMP:
Scott McMillan614f60f2010-06-02 05:53:56 -07003899 val = po->tp_tstamp;
Scott McMillan614f60f2010-06-02 05:53:56 -07003900 break;
David S. Millerdc99f602011-07-05 01:45:05 -07003901 case PACKET_FANOUT:
David S. Millerdc99f602011-07-05 01:45:05 -07003902 val = (po->fanout ?
3903 ((u32)po->fanout->id |
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00003904 ((u32)po->fanout->type << 16) |
3905 ((u32)po->fanout->flags << 24)) :
David S. Millerdc99f602011-07-05 01:45:05 -07003906 0);
David S. Millerdc99f602011-07-05 01:45:05 -07003907 break;
Willem de Bruijna9b63912015-05-12 11:56:50 -04003908 case PACKET_ROLLOVER_STATS:
3909 if (!po->rollover)
3910 return -EINVAL;
3911 rstats.tp_all = atomic_long_read(&po->rollover->num);
3912 rstats.tp_huge = atomic_long_read(&po->rollover->num_huge);
3913 rstats.tp_failed = atomic_long_read(&po->rollover->num_failed);
3914 data = &rstats;
3915 lv = sizeof(rstats);
3916 break;
Paul Chavent5920cd3a2012-11-06 23:10:47 +00003917 case PACKET_TX_HAS_OFF:
3918 val = po->tp_tx_has_off;
3919 break;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01003920 case PACKET_QDISC_BYPASS:
3921 val = packet_use_direct_xmit(po);
3922 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003923 default:
3924 return -ENOPROTOOPT;
3925 }
3926
Eric Dumazetc06fff62012-04-19 21:56:11 +00003927 if (len > lv)
3928 len = lv;
Kris Katterjohn8ae55f02006-01-23 16:28:02 -08003929 if (put_user(len, optlen))
3930 return -EFAULT;
Herbert Xu8dc41942007-02-04 23:31:32 -08003931 if (copy_to_user(optval, data, len))
3932 return -EFAULT;
Kris Katterjohn8ae55f02006-01-23 16:28:02 -08003933 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003934}
3935
3936
Willem de Bruijn719c44d2016-06-07 12:06:34 -04003937#ifdef CONFIG_COMPAT
3938static int compat_packet_setsockopt(struct socket *sock, int level, int optname,
3939 char __user *optval, unsigned int optlen)
3940{
3941 struct packet_sock *po = pkt_sk(sock->sk);
3942
3943 if (level != SOL_PACKET)
3944 return -ENOPROTOOPT;
3945
3946 if (optname == PACKET_FANOUT_DATA &&
3947 po->fanout && po->fanout->type == PACKET_FANOUT_CBPF) {
3948 optval = (char __user *)get_compat_bpf_fprog(optval);
3949 if (!optval)
3950 return -EFAULT;
3951 optlen = sizeof(struct sock_fprog);
3952 }
3953
3954 return packet_setsockopt(sock, level, optname, optval, optlen);
3955}
3956#endif
3957
Jiri Pirko351638e2013-05-28 01:30:21 +00003958static int packet_notifier(struct notifier_block *this,
3959 unsigned long msg, void *ptr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003960{
3961 struct sock *sk;
Jiri Pirko351638e2013-05-28 01:30:21 +00003962 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09003963 struct net *net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003964
stephen hemminger808f5112010-02-22 07:57:18 +00003965 rcu_read_lock();
Sasha Levinb67bfe02013-02-27 17:06:00 -08003966 sk_for_each_rcu(sk, &net->packet.sklist) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003967 struct packet_sock *po = pkt_sk(sk);
3968
3969 switch (msg) {
3970 case NETDEV_UNREGISTER:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003971 if (po->mclist)
Francesco Ruggeri82f17092015-03-09 11:51:04 -07003972 packet_dev_mclist_delete(dev, &po->mclist);
David S. Millera2efcfa2007-05-29 13:12:50 -07003973 /* fallthrough */
3974
Linus Torvalds1da177e2005-04-16 15:20:36 -07003975 case NETDEV_DOWN:
3976 if (dev->ifindex == po->ifindex) {
3977 spin_lock(&po->bind_lock);
3978 if (po->running) {
David S. Millerce06b032011-07-04 01:44:29 -07003979 __unregister_prot_hook(sk, false);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003980 sk->sk_err = ENETDOWN;
3981 if (!sock_flag(sk, SOCK_DEAD))
3982 sk->sk_error_report(sk);
3983 }
3984 if (msg == NETDEV_UNREGISTER) {
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01003985 packet_cached_dev_reset(po);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003986 po->ifindex = -1;
Ben Greear160ff182011-06-01 07:18:52 +00003987 if (po->prot_hook.dev)
3988 dev_put(po->prot_hook.dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003989 po->prot_hook.dev = NULL;
3990 }
3991 spin_unlock(&po->bind_lock);
3992 }
3993 break;
3994 case NETDEV_UP:
stephen hemminger808f5112010-02-22 07:57:18 +00003995 if (dev->ifindex == po->ifindex) {
3996 spin_lock(&po->bind_lock);
David S. Millerce06b032011-07-04 01:44:29 -07003997 if (po->num)
3998 register_prot_hook(sk);
stephen hemminger808f5112010-02-22 07:57:18 +00003999 spin_unlock(&po->bind_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004000 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004001 break;
4002 }
4003 }
stephen hemminger808f5112010-02-22 07:57:18 +00004004 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004005 return NOTIFY_DONE;
4006}
4007
4008
4009static int packet_ioctl(struct socket *sock, unsigned int cmd,
4010 unsigned long arg)
4011{
4012 struct sock *sk = sock->sk;
4013
Johann Baudy69e3c752009-05-18 22:11:22 -07004014 switch (cmd) {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00004015 case SIOCOUTQ:
4016 {
4017 int amount = sk_wmem_alloc_get(sk);
Eric Dumazet31e6d362009-06-17 19:05:41 -07004018
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00004019 return put_user(amount, (int __user *)arg);
4020 }
4021 case SIOCINQ:
4022 {
4023 struct sk_buff *skb;
4024 int amount = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004025
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00004026 spin_lock_bh(&sk->sk_receive_queue.lock);
4027 skb = skb_peek(&sk->sk_receive_queue);
4028 if (skb)
4029 amount = skb->len;
4030 spin_unlock_bh(&sk->sk_receive_queue.lock);
4031 return put_user(amount, (int __user *)arg);
4032 }
4033 case SIOCGSTAMP:
4034 return sock_get_timestamp(sk, (struct timeval __user *)arg);
4035 case SIOCGSTAMPNS:
4036 return sock_get_timestampns(sk, (struct timespec __user *)arg);
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09004037
Linus Torvalds1da177e2005-04-16 15:20:36 -07004038#ifdef CONFIG_INET
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00004039 case SIOCADDRT:
4040 case SIOCDELRT:
4041 case SIOCDARP:
4042 case SIOCGARP:
4043 case SIOCSARP:
4044 case SIOCGIFADDR:
4045 case SIOCSIFADDR:
4046 case SIOCGIFBRDADDR:
4047 case SIOCSIFBRDADDR:
4048 case SIOCGIFNETMASK:
4049 case SIOCSIFNETMASK:
4050 case SIOCGIFDSTADDR:
4051 case SIOCSIFDSTADDR:
4052 case SIOCSIFFLAGS:
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00004053 return inet_dgram_ops.ioctl(sock, cmd, arg);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004054#endif
4055
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00004056 default:
4057 return -ENOIOCTLCMD;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004058 }
4059 return 0;
4060}
4061
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00004062static unsigned int packet_poll(struct file *file, struct socket *sock,
Linus Torvalds1da177e2005-04-16 15:20:36 -07004063 poll_table *wait)
4064{
4065 struct sock *sk = sock->sk;
4066 struct packet_sock *po = pkt_sk(sk);
4067 unsigned int mask = datagram_poll(file, sock, wait);
4068
4069 spin_lock_bh(&sk->sk_receive_queue.lock);
Johann Baudy69e3c752009-05-18 22:11:22 -07004070 if (po->rx_ring.pg_vec) {
chetan lokef6fb8f102011-08-19 10:18:16 +00004071 if (!packet_previous_rx_frame(po, &po->rx_ring,
4072 TP_STATUS_KERNEL))
Linus Torvalds1da177e2005-04-16 15:20:36 -07004073 mask |= POLLIN | POLLRDNORM;
4074 }
Willem de Bruijn2ccdbaa2015-05-12 11:56:48 -04004075 if (po->pressure && __packet_rcv_has_room(po, NULL) == ROOM_NORMAL)
Willem de Bruijn54d7c012015-05-14 15:25:02 -04004076 po->pressure = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004077 spin_unlock_bh(&sk->sk_receive_queue.lock);
Johann Baudy69e3c752009-05-18 22:11:22 -07004078 spin_lock_bh(&sk->sk_write_queue.lock);
4079 if (po->tx_ring.pg_vec) {
4080 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
4081 mask |= POLLOUT | POLLWRNORM;
4082 }
4083 spin_unlock_bh(&sk->sk_write_queue.lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004084 return mask;
4085}
4086
4087
4088/* Dirty? Well, I still did not learn better way to account
4089 * for user mmaps.
4090 */
4091
4092static void packet_mm_open(struct vm_area_struct *vma)
4093{
4094 struct file *file = vma->vm_file;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00004095 struct socket *sock = file->private_data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004096 struct sock *sk = sock->sk;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09004097
Linus Torvalds1da177e2005-04-16 15:20:36 -07004098 if (sk)
4099 atomic_inc(&pkt_sk(sk)->mapped);
4100}
4101
4102static void packet_mm_close(struct vm_area_struct *vma)
4103{
4104 struct file *file = vma->vm_file;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00004105 struct socket *sock = file->private_data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004106 struct sock *sk = sock->sk;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09004107
Linus Torvalds1da177e2005-04-16 15:20:36 -07004108 if (sk)
4109 atomic_dec(&pkt_sk(sk)->mapped);
4110}
4111
Alexey Dobriyanf0f37e22009-09-27 22:29:37 +04004112static const struct vm_operations_struct packet_mmap_ops = {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00004113 .open = packet_mm_open,
4114 .close = packet_mm_close,
Linus Torvalds1da177e2005-04-16 15:20:36 -07004115};
4116
Neil Horman0e3125c2010-11-16 10:26:47 -08004117static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
4118 unsigned int len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004119{
4120 int i;
4121
David S. Miller4ebf0ae2005-12-06 16:38:35 -08004122 for (i = 0; i < len; i++) {
Neil Horman0e3125c2010-11-16 10:26:47 -08004123 if (likely(pg_vec[i].buffer)) {
Changli Gaoc56b4d92010-12-01 02:52:57 +00004124 if (is_vmalloc_addr(pg_vec[i].buffer))
Neil Horman0e3125c2010-11-16 10:26:47 -08004125 vfree(pg_vec[i].buffer);
4126 else
4127 free_pages((unsigned long)pg_vec[i].buffer,
4128 order);
4129 pg_vec[i].buffer = NULL;
4130 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004131 }
4132 kfree(pg_vec);
4133}
4134
Olof Johanssoneea49cc92011-11-02 11:00:49 +00004135static char *alloc_one_pg_vec_page(unsigned long order)
David S. Miller4ebf0ae2005-12-06 16:38:35 -08004136{
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01004137 char *buffer;
Neil Horman0e3125c2010-11-16 10:26:47 -08004138 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
4139 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
Eric Dumazet719bfea2009-04-15 03:39:52 -07004140
Neil Horman0e3125c2010-11-16 10:26:47 -08004141 buffer = (char *) __get_free_pages(gfp_flags, order);
Neil Horman0e3125c2010-11-16 10:26:47 -08004142 if (buffer)
4143 return buffer;
4144
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01004145 /* __get_free_pages failed, fall back to vmalloc */
Eric Dumazetbbce5a52010-11-20 07:31:54 +00004146 buffer = vzalloc((1 << order) * PAGE_SIZE);
Neil Horman0e3125c2010-11-16 10:26:47 -08004147 if (buffer)
4148 return buffer;
4149
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01004150 /* vmalloc failed, lets dig into swap here */
Neil Horman0e3125c2010-11-16 10:26:47 -08004151 gfp_flags &= ~__GFP_NORETRY;
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01004152 buffer = (char *) __get_free_pages(gfp_flags, order);
Neil Horman0e3125c2010-11-16 10:26:47 -08004153 if (buffer)
4154 return buffer;
4155
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01004156 /* complete and utter failure */
Neil Horman0e3125c2010-11-16 10:26:47 -08004157 return NULL;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08004158}
4159
Neil Horman0e3125c2010-11-16 10:26:47 -08004160static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
David S. Miller4ebf0ae2005-12-06 16:38:35 -08004161{
4162 unsigned int block_nr = req->tp_block_nr;
Neil Horman0e3125c2010-11-16 10:26:47 -08004163 struct pgv *pg_vec;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08004164 int i;
4165
Neil Horman0e3125c2010-11-16 10:26:47 -08004166 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
David S. Miller4ebf0ae2005-12-06 16:38:35 -08004167 if (unlikely(!pg_vec))
4168 goto out;
4169
4170 for (i = 0; i < block_nr; i++) {
Changli Gaoc56b4d92010-12-01 02:52:57 +00004171 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
Neil Horman0e3125c2010-11-16 10:26:47 -08004172 if (unlikely(!pg_vec[i].buffer))
David S. Miller4ebf0ae2005-12-06 16:38:35 -08004173 goto out_free_pgvec;
4174 }
4175
4176out:
4177 return pg_vec;
4178
4179out_free_pgvec:
4180 free_pg_vec(pg_vec, order, block_nr);
4181 pg_vec = NULL;
4182 goto out;
4183}
Linus Torvalds1da177e2005-04-16 15:20:36 -07004184
chetan lokef6fb8f102011-08-19 10:18:16 +00004185static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
Johann Baudy69e3c752009-05-18 22:11:22 -07004186 int closing, int tx_ring)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004187{
Neil Horman0e3125c2010-11-16 10:26:47 -08004188 struct pgv *pg_vec = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004189 struct packet_sock *po = pkt_sk(sk);
Al Viro0e11c912006-11-08 00:26:29 -08004190 int was_running, order = 0;
Johann Baudy69e3c752009-05-18 22:11:22 -07004191 struct packet_ring_buffer *rb;
4192 struct sk_buff_head *rb_queue;
Al Viro0e11c912006-11-08 00:26:29 -08004193 __be16 num;
chetan lokef6fb8f102011-08-19 10:18:16 +00004194 int err = -EINVAL;
4195 /* Added to avoid minimal code churn */
4196 struct tpacket_req *req = &req_u->req;
4197
Philip Pettersson84ac7262016-11-30 14:55:36 -08004198 lock_sock(sk);
chetan lokef6fb8f102011-08-19 10:18:16 +00004199 /* Opening a Tx-ring is NOT supported in TPACKET_V3 */
4200 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) {
Dave Jones6ae81ce2016-04-04 15:11:50 -04004201 net_warn_ratelimited("Tx-ring is not supported.\n");
chetan lokef6fb8f102011-08-19 10:18:16 +00004202 goto out;
4203 }
Johann Baudy69e3c752009-05-18 22:11:22 -07004204
4205 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
4206 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
4207
4208 err = -EBUSY;
4209 if (!closing) {
4210 if (atomic_read(&po->mapped))
4211 goto out;
Daniel Borkmannb0138402014-01-15 16:25:36 +01004212 if (packet_read_pending(rb))
Johann Baudy69e3c752009-05-18 22:11:22 -07004213 goto out;
4214 }
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09004215
Linus Torvalds1da177e2005-04-16 15:20:36 -07004216 if (req->tp_block_nr) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004217 /* Sanity tests and some calculations */
Johann Baudy69e3c752009-05-18 22:11:22 -07004218 err = -EBUSY;
4219 if (unlikely(rb->pg_vec))
4220 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004221
Patrick McHardybbd6ef82008-07-14 22:50:15 -07004222 switch (po->tp_version) {
4223 case TPACKET_V1:
4224 po->tp_hdrlen = TPACKET_HDRLEN;
4225 break;
4226 case TPACKET_V2:
4227 po->tp_hdrlen = TPACKET2_HDRLEN;
4228 break;
chetan lokef6fb8f102011-08-19 10:18:16 +00004229 case TPACKET_V3:
4230 po->tp_hdrlen = TPACKET3_HDRLEN;
4231 break;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07004232 }
4233
Johann Baudy69e3c752009-05-18 22:11:22 -07004234 err = -EINVAL;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08004235 if (unlikely((int)req->tp_block_size <= 0))
Johann Baudy69e3c752009-05-18 22:11:22 -07004236 goto out;
Tobias Klauser90836b62015-11-17 10:40:21 +01004237 if (unlikely(!PAGE_ALIGNED(req->tp_block_size)))
Johann Baudy69e3c752009-05-18 22:11:22 -07004238 goto out;
Eric Dumazetdc808112014-08-15 09:16:04 -07004239 if (po->tp_version >= TPACKET_V3 &&
Andrey Konovalov16fc98c2017-03-29 16:11:20 +02004240 req->tp_block_size <=
4241 BLK_PLUS_PRIV((u64)req_u->req3.tp_sizeof_priv))
Eric Dumazetdc808112014-08-15 09:16:04 -07004242 goto out;
Patrick McHardy8913336a2008-07-18 18:05:19 -07004243 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
Johann Baudy69e3c752009-05-18 22:11:22 -07004244 po->tp_reserve))
4245 goto out;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08004246 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
Johann Baudy69e3c752009-05-18 22:11:22 -07004247 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004248
Tobias Klauser4194b492015-11-17 10:38:36 +01004249 rb->frames_per_block = req->tp_block_size / req->tp_frame_size;
4250 if (unlikely(rb->frames_per_block == 0))
Johann Baudy69e3c752009-05-18 22:11:22 -07004251 goto out;
Andrey Konovalov10452122017-03-29 16:11:21 +02004252 if (unlikely(req->tp_block_size > UINT_MAX / req->tp_block_nr))
4253 goto out;
Johann Baudy69e3c752009-05-18 22:11:22 -07004254 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
4255 req->tp_frame_nr))
4256 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004257
4258 err = -ENOMEM;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08004259 order = get_order(req->tp_block_size);
4260 pg_vec = alloc_pg_vec(req, order);
4261 if (unlikely(!pg_vec))
Linus Torvalds1da177e2005-04-16 15:20:36 -07004262 goto out;
chetan lokef6fb8f102011-08-19 10:18:16 +00004263 switch (po->tp_version) {
4264 case TPACKET_V3:
4265 /* Transmit path is not supported. We checked
4266 * it above but just being paranoid
4267 */
4268 if (!tx_ring)
Maninder Singhe8e85cc2015-06-22 12:39:16 +05304269 init_prb_bdqc(po, rb, pg_vec, req_u);
Dan Carpenterd7cf0c32014-02-18 15:20:51 +03004270 break;
chetan lokef6fb8f102011-08-19 10:18:16 +00004271 default:
4272 break;
4273 }
Johann Baudy69e3c752009-05-18 22:11:22 -07004274 }
4275 /* Done */
4276 else {
4277 err = -EINVAL;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08004278 if (unlikely(req->tp_frame_nr))
Johann Baudy69e3c752009-05-18 22:11:22 -07004279 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004280 }
4281
Linus Torvalds1da177e2005-04-16 15:20:36 -07004282
4283 /* Detach socket from network */
4284 spin_lock(&po->bind_lock);
4285 was_running = po->running;
4286 num = po->num;
4287 if (was_running) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004288 po->num = 0;
David S. Millerce06b032011-07-04 01:44:29 -07004289 __unregister_prot_hook(sk, false);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004290 }
4291 spin_unlock(&po->bind_lock);
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09004292
Linus Torvalds1da177e2005-04-16 15:20:36 -07004293 synchronize_net();
4294
4295 err = -EBUSY;
Herbert Xu905db442009-01-30 14:12:06 -08004296 mutex_lock(&po->pg_vec_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004297 if (closing || atomic_read(&po->mapped) == 0) {
4298 err = 0;
Johann Baudy69e3c752009-05-18 22:11:22 -07004299 spin_lock_bh(&rb_queue->lock);
Changli Gaoc053fd92010-12-10 16:02:20 -08004300 swap(rb->pg_vec, pg_vec);
Johann Baudy69e3c752009-05-18 22:11:22 -07004301 rb->frame_max = (req->tp_frame_nr - 1);
4302 rb->head = 0;
4303 rb->frame_size = req->tp_frame_size;
4304 spin_unlock_bh(&rb_queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004305
Changli Gaoc053fd92010-12-10 16:02:20 -08004306 swap(rb->pg_vec_order, order);
4307 swap(rb->pg_vec_len, req->tp_block_nr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004308
Johann Baudy69e3c752009-05-18 22:11:22 -07004309 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
4310 po->prot_hook.func = (po->rx_ring.pg_vec) ?
4311 tpacket_rcv : packet_rcv;
4312 skb_queue_purge(rb_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004313 if (atomic_read(&po->mapped))
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00004314 pr_err("packet_mmap: vma is busy: %d\n",
4315 atomic_read(&po->mapped));
Linus Torvalds1da177e2005-04-16 15:20:36 -07004316 }
Herbert Xu905db442009-01-30 14:12:06 -08004317 mutex_unlock(&po->pg_vec_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004318
4319 spin_lock(&po->bind_lock);
David S. Millerce06b032011-07-04 01:44:29 -07004320 if (was_running) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004321 po->num = num;
David S. Millerce06b032011-07-04 01:44:29 -07004322 register_prot_hook(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004323 }
4324 spin_unlock(&po->bind_lock);
chetan lokef6fb8f102011-08-19 10:18:16 +00004325 if (closing && (po->tp_version > TPACKET_V2)) {
4326 /* Because we don't support block-based V3 on tx-ring */
4327 if (!tx_ring)
Tobias Klauser73d0fcf2015-07-28 14:21:26 +02004328 prb_shutdown_retire_blk_timer(po, rb_queue);
chetan lokef6fb8f102011-08-19 10:18:16 +00004329 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004330
Linus Torvalds1da177e2005-04-16 15:20:36 -07004331 if (pg_vec)
4332 free_pg_vec(pg_vec, order, req->tp_block_nr);
4333out:
Philip Pettersson84ac7262016-11-30 14:55:36 -08004334 release_sock(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004335 return err;
4336}
4337
Johann Baudy69e3c752009-05-18 22:11:22 -07004338static int packet_mmap(struct file *file, struct socket *sock,
4339 struct vm_area_struct *vma)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004340{
4341 struct sock *sk = sock->sk;
4342 struct packet_sock *po = pkt_sk(sk);
Johann Baudy69e3c752009-05-18 22:11:22 -07004343 unsigned long size, expected_size;
4344 struct packet_ring_buffer *rb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004345 unsigned long start;
4346 int err = -EINVAL;
4347 int i;
4348
4349 if (vma->vm_pgoff)
4350 return -EINVAL;
4351
Herbert Xu905db442009-01-30 14:12:06 -08004352 mutex_lock(&po->pg_vec_lock);
Johann Baudy69e3c752009-05-18 22:11:22 -07004353
4354 expected_size = 0;
4355 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
4356 if (rb->pg_vec) {
4357 expected_size += rb->pg_vec_len
4358 * rb->pg_vec_pages
4359 * PAGE_SIZE;
4360 }
4361 }
4362
4363 if (expected_size == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004364 goto out;
Johann Baudy69e3c752009-05-18 22:11:22 -07004365
4366 size = vma->vm_end - vma->vm_start;
4367 if (size != expected_size)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004368 goto out;
4369
Linus Torvalds1da177e2005-04-16 15:20:36 -07004370 start = vma->vm_start;
Johann Baudy69e3c752009-05-18 22:11:22 -07004371 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
4372 if (rb->pg_vec == NULL)
4373 continue;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08004374
Johann Baudy69e3c752009-05-18 22:11:22 -07004375 for (i = 0; i < rb->pg_vec_len; i++) {
Neil Horman0e3125c2010-11-16 10:26:47 -08004376 struct page *page;
4377 void *kaddr = rb->pg_vec[i].buffer;
Johann Baudy69e3c752009-05-18 22:11:22 -07004378 int pg_num;
4379
Changli Gaoc56b4d92010-12-01 02:52:57 +00004380 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
4381 page = pgv_to_page(kaddr);
Johann Baudy69e3c752009-05-18 22:11:22 -07004382 err = vm_insert_page(vma, start, page);
4383 if (unlikely(err))
4384 goto out;
4385 start += PAGE_SIZE;
Neil Horman0e3125c2010-11-16 10:26:47 -08004386 kaddr += PAGE_SIZE;
Johann Baudy69e3c752009-05-18 22:11:22 -07004387 }
David S. Miller4ebf0ae2005-12-06 16:38:35 -08004388 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004389 }
Johann Baudy69e3c752009-05-18 22:11:22 -07004390
David S. Miller4ebf0ae2005-12-06 16:38:35 -08004391 atomic_inc(&po->mapped);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004392 vma->vm_ops = &packet_mmap_ops;
4393 err = 0;
4394
4395out:
Herbert Xu905db442009-01-30 14:12:06 -08004396 mutex_unlock(&po->pg_vec_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004397 return err;
4398}
Linus Torvalds1da177e2005-04-16 15:20:36 -07004399
Eric Dumazet90ddc4f2005-12-22 12:49:22 -08004400static const struct proto_ops packet_ops_spkt = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004401 .family = PF_PACKET,
4402 .owner = THIS_MODULE,
4403 .release = packet_release,
4404 .bind = packet_bind_spkt,
4405 .connect = sock_no_connect,
4406 .socketpair = sock_no_socketpair,
4407 .accept = sock_no_accept,
4408 .getname = packet_getname_spkt,
4409 .poll = datagram_poll,
4410 .ioctl = packet_ioctl,
4411 .listen = sock_no_listen,
4412 .shutdown = sock_no_shutdown,
4413 .setsockopt = sock_no_setsockopt,
4414 .getsockopt = sock_no_getsockopt,
4415 .sendmsg = packet_sendmsg_spkt,
4416 .recvmsg = packet_recvmsg,
4417 .mmap = sock_no_mmap,
4418 .sendpage = sock_no_sendpage,
4419};
Linus Torvalds1da177e2005-04-16 15:20:36 -07004420
Eric Dumazet90ddc4f2005-12-22 12:49:22 -08004421static const struct proto_ops packet_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004422 .family = PF_PACKET,
4423 .owner = THIS_MODULE,
4424 .release = packet_release,
4425 .bind = packet_bind,
4426 .connect = sock_no_connect,
4427 .socketpair = sock_no_socketpair,
4428 .accept = sock_no_accept,
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09004429 .getname = packet_getname,
Linus Torvalds1da177e2005-04-16 15:20:36 -07004430 .poll = packet_poll,
4431 .ioctl = packet_ioctl,
4432 .listen = sock_no_listen,
4433 .shutdown = sock_no_shutdown,
4434 .setsockopt = packet_setsockopt,
4435 .getsockopt = packet_getsockopt,
Willem de Bruijn719c44d2016-06-07 12:06:34 -04004436#ifdef CONFIG_COMPAT
4437 .compat_setsockopt = compat_packet_setsockopt,
4438#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07004439 .sendmsg = packet_sendmsg,
4440 .recvmsg = packet_recvmsg,
4441 .mmap = packet_mmap,
4442 .sendpage = sock_no_sendpage,
4443};
4444
Stephen Hemmingerec1b4cf2009-10-05 05:58:39 +00004445static const struct net_proto_family packet_family_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004446 .family = PF_PACKET,
4447 .create = packet_create,
4448 .owner = THIS_MODULE,
4449};
4450
4451static struct notifier_block packet_netdev_notifier = {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00004452 .notifier_call = packet_notifier,
Linus Torvalds1da177e2005-04-16 15:20:36 -07004453};
4454
4455#ifdef CONFIG_PROC_FS
Linus Torvalds1da177e2005-04-16 15:20:36 -07004456
4457static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
stephen hemminger808f5112010-02-22 07:57:18 +00004458 __acquires(RCU)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004459{
Denis V. Luneve372c412007-11-19 22:31:54 -08004460 struct net *net = seq_file_net(seq);
stephen hemminger808f5112010-02-22 07:57:18 +00004461
4462 rcu_read_lock();
4463 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004464}
4465
4466static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4467{
Herbert Xu1bf40952007-12-16 14:04:02 -08004468 struct net *net = seq_file_net(seq);
stephen hemminger808f5112010-02-22 07:57:18 +00004469 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004470}
4471
4472static void packet_seq_stop(struct seq_file *seq, void *v)
stephen hemminger808f5112010-02-22 07:57:18 +00004473 __releases(RCU)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004474{
stephen hemminger808f5112010-02-22 07:57:18 +00004475 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004476}
4477
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09004478static int packet_seq_show(struct seq_file *seq, void *v)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004479{
4480 if (v == SEQ_START_TOKEN)
4481 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
4482 else {
Li Zefanb7ceabd2010-02-08 23:19:29 +00004483 struct sock *s = sk_entry(v);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004484 const struct packet_sock *po = pkt_sk(s);
4485
4486 seq_printf(seq,
Dan Rosenberg71338aa2011-05-23 12:17:35 +00004487 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07004488 s,
4489 atomic_read(&s->sk_refcnt),
4490 s->sk_type,
4491 ntohs(po->num),
4492 po->ifindex,
4493 po->running,
4494 atomic_read(&s->sk_rmem_alloc),
Eric W. Biedermana7cb5a42012-05-24 01:10:10 -06004495 from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00004496 sock_i_ino(s));
Linus Torvalds1da177e2005-04-16 15:20:36 -07004497 }
4498
4499 return 0;
4500}
4501
Philippe De Muyter56b3d972007-07-10 23:07:31 -07004502static const struct seq_operations packet_seq_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004503 .start = packet_seq_start,
4504 .next = packet_seq_next,
4505 .stop = packet_seq_stop,
4506 .show = packet_seq_show,
4507};
4508
4509static int packet_seq_open(struct inode *inode, struct file *file)
4510{
Denis V. Luneve372c412007-11-19 22:31:54 -08004511 return seq_open_net(inode, file, &packet_seq_ops,
4512 sizeof(struct seq_net_private));
Linus Torvalds1da177e2005-04-16 15:20:36 -07004513}
4514
Arjan van de Venda7071d2007-02-12 00:55:36 -08004515static const struct file_operations packet_seq_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004516 .owner = THIS_MODULE,
4517 .open = packet_seq_open,
4518 .read = seq_read,
4519 .llseek = seq_lseek,
Denis V. Luneve372c412007-11-19 22:31:54 -08004520 .release = seq_release_net,
Linus Torvalds1da177e2005-04-16 15:20:36 -07004521};
4522
4523#endif
4524
Alexey Dobriyan2c8c1e72010-01-17 03:35:32 +00004525static int __net_init packet_net_init(struct net *net)
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004526{
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00004527 mutex_init(&net->packet.sklist_lock);
Denis V. Lunev2aaef4e2007-12-11 04:19:54 -08004528 INIT_HLIST_HEAD(&net->packet.sklist);
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004529
Gao fengd4beaa62013-02-18 01:34:54 +00004530 if (!proc_create("packet", 0, net->proc_net, &packet_seq_fops))
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004531 return -ENOMEM;
4532
4533 return 0;
4534}
4535
Alexey Dobriyan2c8c1e72010-01-17 03:35:32 +00004536static void __net_exit packet_net_exit(struct net *net)
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004537{
Gao fengece31ff2013-02-18 01:34:56 +00004538 remove_proc_entry("packet", net->proc_net);
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004539}
4540
4541static struct pernet_operations packet_net_ops = {
4542 .init = packet_net_init,
4543 .exit = packet_net_exit,
4544};
4545
4546
Linus Torvalds1da177e2005-04-16 15:20:36 -07004547static void __exit packet_exit(void)
4548{
Linus Torvalds1da177e2005-04-16 15:20:36 -07004549 unregister_netdevice_notifier(&packet_netdev_notifier);
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004550 unregister_pernet_subsys(&packet_net_ops);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004551 sock_unregister(PF_PACKET);
4552 proto_unregister(&packet_proto);
4553}
4554
4555static int __init packet_init(void)
4556{
4557 int rc = proto_register(&packet_proto, 0);
4558
4559 if (rc != 0)
4560 goto out;
4561
4562 sock_register(&packet_family_ops);
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004563 register_pernet_subsys(&packet_net_ops);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004564 register_netdevice_notifier(&packet_netdev_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004565out:
4566 return rc;
4567}
4568
4569module_init(packet_init);
4570module_exit(packet_exit);
4571MODULE_LICENSE("GPL");
4572MODULE_ALIAS_NETPROTO(PF_PACKET);