blob: 5102c3cc4eec4ecec6698859935d7769d37a174c [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * PACKET - implements raw packet sockets.
7 *
Jesper Juhl02c30a82005-05-05 16:16:16 -07008 * Authors: Ross Biro
Linus Torvalds1da177e2005-04-16 15:20:36 -07009 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
11 *
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +090012 * Fixes:
Linus Torvalds1da177e2005-04-16 15:20:36 -070013 * Alan Cox : verify_area() now used correctly
14 * Alan Cox : new skbuff lists, look ma no backlogs!
15 * Alan Cox : tidied skbuff lists.
16 * Alan Cox : Now uses generic datagram routines I
17 * added. Also fixed the peek/read crash
18 * from all old Linux datagram code.
19 * Alan Cox : Uses the improved datagram code.
20 * Alan Cox : Added NULL's for socket options.
21 * Alan Cox : Re-commented the code.
22 * Alan Cox : Use new kernel side addressing
23 * Rob Janssen : Correct MTU usage.
24 * Dave Platt : Counter leaks caused by incorrect
25 * interrupt locking and some slightly
26 * dubious gcc output. Can you read
27 * compiler: it said _VOLATILE_
28 * Richard Kooijman : Timestamp fixes.
29 * Alan Cox : New buffers. Use sk->mac.raw.
30 * Alan Cox : sendmsg/recvmsg support.
31 * Alan Cox : Protocol setting support
32 * Alexey Kuznetsov : Untied from IPv4 stack.
33 * Cyrus Durgin : Fixed kerneld for kmod.
34 * Michal Ostrowski : Module initialization cleanup.
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +090035 * Ulises Alonso : Frame number limit removal and
Linus Torvalds1da177e2005-04-16 15:20:36 -070036 * packet_set_ring memory leak.
Eric W. Biederman0fb375f2005-09-21 00:11:37 -070037 * Eric Biederman : Allow for > 8 byte hardware addresses.
38 * The convention is that longer addresses
39 * will simply extend the hardware address
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +090040 * byte arrays at the end of sockaddr_ll
Eric W. Biederman0fb375f2005-09-21 00:11:37 -070041 * and packet_mreq.
Johann Baudy69e3c752009-05-18 22:11:22 -070042 * Johann Baudy : Added TX RING.
chetan lokef6fb8f102011-08-19 10:18:16 +000043 * Chetan Loke : Implemented TPACKET_V3 block abstraction
44 * layer.
45 * Copyright (C) 2011, <lokec@ccs.neu.edu>
46 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070047 *
48 * This program is free software; you can redistribute it and/or
49 * modify it under the terms of the GNU General Public License
50 * as published by the Free Software Foundation; either version
51 * 2 of the License, or (at your option) any later version.
52 *
53 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +090054
Linus Torvalds1da177e2005-04-16 15:20:36 -070055#include <linux/types.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070056#include <linux/mm.h>
Randy Dunlap4fc268d2006-01-11 12:17:47 -080057#include <linux/capability.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070058#include <linux/fcntl.h>
59#include <linux/socket.h>
60#include <linux/in.h>
61#include <linux/inet.h>
62#include <linux/netdevice.h>
63#include <linux/if_packet.h>
64#include <linux/wireless.h>
Herbert Xuffbc6112007-02-04 23:33:10 -080065#include <linux/kernel.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070066#include <linux/kmod.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090067#include <linux/slab.h>
Neil Horman0e3125c2010-11-16 10:26:47 -080068#include <linux/vmalloc.h>
Eric W. Biederman457c4cb2007-09-12 12:01:34 +020069#include <net/net_namespace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070070#include <net/ip.h>
71#include <net/protocol.h>
72#include <linux/skbuff.h>
73#include <net/sock.h>
74#include <linux/errno.h>
75#include <linux/timer.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070076#include <asm/uaccess.h>
77#include <asm/ioctls.h>
78#include <asm/page.h>
Al Viroa1f8e7f72006-10-19 16:08:53 -040079#include <asm/cacheflush.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070080#include <asm/io.h>
81#include <linux/proc_fs.h>
82#include <linux/seq_file.h>
83#include <linux/poll.h>
84#include <linux/module.h>
85#include <linux/init.h>
Herbert Xu905db442009-01-30 14:12:06 -080086#include <linux/mutex.h>
Eric Dumazet05423b22009-10-26 18:40:35 -070087#include <linux/if_vlan.h>
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -080088#include <linux/virtio_net.h>
Richard Cochraned85b562010-04-07 22:41:28 +000089#include <linux/errqueue.h>
Scott McMillan614f60f2010-06-02 05:53:56 -070090#include <linux/net_tstamp.h>
Daniel Borkmannb0138402014-01-15 16:25:36 +010091#include <linux/percpu.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070092#ifdef CONFIG_INET
93#include <net/inet_common.h>
94#endif
95
Pavel Emelyanov2787b042012-08-13 05:49:39 +000096#include "internal.h"
97
Linus Torvalds1da177e2005-04-16 15:20:36 -070098/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070099 Assumptions:
100 - if device has no dev->hard_header routine, it adds and removes ll header
101 inside itself. In this case ll header is invisible outside of device,
102 but higher levels still should reserve dev->hard_header_len.
103 Some devices are enough clever to reallocate skb, when header
104 will not fit to reserved space (tunnel), another ones are silly
105 (PPP).
106 - packet socket receives packets with pulled ll header,
107 so that SOCK_RAW should push it back.
108
109On receive:
110-----------
111
112Incoming, dev->hard_header!=NULL
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -0700113 mac_header -> ll header
114 data -> data
Linus Torvalds1da177e2005-04-16 15:20:36 -0700115
116Outgoing, dev->hard_header!=NULL
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -0700117 mac_header -> ll header
118 data -> ll header
Linus Torvalds1da177e2005-04-16 15:20:36 -0700119
120Incoming, dev->hard_header==NULL
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -0700121 mac_header -> UNKNOWN position. It is very likely, that it points to ll
122 header. PPP makes it, that is wrong, because introduce
YOSHIFUJI Hideakidb0c58f2007-07-19 10:44:35 +0900123 assymetry between rx and tx paths.
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -0700124 data -> data
Linus Torvalds1da177e2005-04-16 15:20:36 -0700125
126Outgoing, dev->hard_header==NULL
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -0700127 mac_header -> data. ll header is still not built!
128 data -> data
Linus Torvalds1da177e2005-04-16 15:20:36 -0700129
130Resume
131 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
132
133
134On transmit:
135------------
136
137dev->hard_header != NULL
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -0700138 mac_header -> ll header
139 data -> ll header
Linus Torvalds1da177e2005-04-16 15:20:36 -0700140
141dev->hard_header == NULL (ll header is added by device, we cannot control it)
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -0700142 mac_header -> data
143 data -> data
Linus Torvalds1da177e2005-04-16 15:20:36 -0700144
145 We should set nh.raw on output to correct posistion,
146 packet classifier depends on it.
147 */
148
Linus Torvalds1da177e2005-04-16 15:20:36 -0700149/* Private packet socket structures. */
150
Eric W. Biederman0fb375f2005-09-21 00:11:37 -0700151/* identical to struct packet_mreq except it has
152 * a longer address field.
153 */
Eric Dumazet40d4e3d2009-07-21 21:57:59 +0000154struct packet_mreq_max {
Eric W. Biederman0fb375f2005-09-21 00:11:37 -0700155 int mr_ifindex;
156 unsigned short mr_type;
157 unsigned short mr_alen;
158 unsigned char mr_address[MAX_ADDR_LEN];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700159};
David S. Millera2efcfa2007-05-29 13:12:50 -0700160
Daniel Borkmann184f4892013-04-16 01:57:46 +0000161union tpacket_uhdr {
162 struct tpacket_hdr *h1;
163 struct tpacket2_hdr *h2;
164 struct tpacket3_hdr *h3;
165 void *raw;
166};
167
chetan lokef6fb8f102011-08-19 10:18:16 +0000168static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
Johann Baudy69e3c752009-05-18 22:11:22 -0700169 int closing, int tx_ring);
170
chetan lokef6fb8f102011-08-19 10:18:16 +0000171#define V3_ALIGNMENT (8)
172
chetan lokebc59ba32011-08-25 10:43:30 +0000173#define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
chetan lokef6fb8f102011-08-19 10:18:16 +0000174
175#define BLK_PLUS_PRIV(sz_of_priv) \
176 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
177
chetan lokef6fb8f102011-08-19 10:18:16 +0000178#define PGV_FROM_VMALLOC 1
Johann Baudy69e3c752009-05-18 22:11:22 -0700179
chetan lokef6fb8f102011-08-19 10:18:16 +0000180#define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
181#define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
182#define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
183#define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
184#define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
185#define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
186#define BLOCK_PRIV(x) ((void *)((char *)(x) + BLOCK_O2PRIV(x)))
187
Johann Baudy69e3c752009-05-18 22:11:22 -0700188struct packet_sock;
189static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
Willem de Bruijn77f65eb2013-03-19 10:18:11 +0000190static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
191 struct packet_type *pt, struct net_device *orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700192
chetan lokef6fb8f102011-08-19 10:18:16 +0000193static void *packet_previous_frame(struct packet_sock *po,
194 struct packet_ring_buffer *rb,
195 int status);
196static void packet_increment_head(struct packet_ring_buffer *buff);
chetan lokebc59ba32011-08-25 10:43:30 +0000197static int prb_curr_blk_in_use(struct tpacket_kbdq_core *,
198 struct tpacket_block_desc *);
199static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
chetan lokef6fb8f102011-08-19 10:18:16 +0000200 struct packet_sock *);
chetan lokebc59ba32011-08-25 10:43:30 +0000201static void prb_retire_current_block(struct tpacket_kbdq_core *,
chetan lokef6fb8f102011-08-19 10:18:16 +0000202 struct packet_sock *, unsigned int status);
chetan lokebc59ba32011-08-25 10:43:30 +0000203static int prb_queue_frozen(struct tpacket_kbdq_core *);
204static void prb_open_block(struct tpacket_kbdq_core *,
205 struct tpacket_block_desc *);
chetan lokef6fb8f102011-08-19 10:18:16 +0000206static void prb_retire_rx_blk_timer_expired(unsigned long);
chetan lokebc59ba32011-08-25 10:43:30 +0000207static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
208static void prb_init_blk_timer(struct packet_sock *,
209 struct tpacket_kbdq_core *,
210 void (*func) (unsigned long));
211static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
212static void prb_clear_rxhash(struct tpacket_kbdq_core *,
213 struct tpacket3_hdr *);
214static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
215 struct tpacket3_hdr *);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700216static void packet_flush_mclist(struct sock *sk);
217
Herbert Xuffbc6112007-02-04 23:33:10 -0800218struct packet_skb_cb {
Herbert Xuffbc6112007-02-04 23:33:10 -0800219 union {
220 struct sockaddr_pkt pkt;
Eyal Birger2472d762015-03-01 14:58:28 +0200221 union {
222 /* Trick: alias skb original length with
223 * ll.sll_family and ll.protocol in order
224 * to save room.
225 */
226 unsigned int origlen;
227 struct sockaddr_ll ll;
228 };
Herbert Xuffbc6112007-02-04 23:33:10 -0800229 } sa;
230};
231
232#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
Herbert Xu8dc41942007-02-04 23:31:32 -0800233
chetan lokebc59ba32011-08-25 10:43:30 +0000234#define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
chetan lokef6fb8f102011-08-19 10:18:16 +0000235#define GET_PBLOCK_DESC(x, bid) \
chetan lokebc59ba32011-08-25 10:43:30 +0000236 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
chetan lokef6fb8f102011-08-19 10:18:16 +0000237#define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
chetan lokebc59ba32011-08-25 10:43:30 +0000238 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
chetan lokef6fb8f102011-08-19 10:18:16 +0000239#define GET_NEXT_PRB_BLK_NUM(x) \
240 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
241 ((x)->kactive_blk_num+1) : 0)
242
David S. Millerdc99f602011-07-05 01:45:05 -0700243static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
244static void __fanout_link(struct sock *sk, struct packet_sock *po);
245
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100246static int packet_direct_xmit(struct sk_buff *skb)
247{
248 struct net_device *dev = skb->dev;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100249 netdev_features_t features;
250 struct netdev_queue *txq;
Daniel Borkmann43279502014-03-27 16:38:30 +0100251 int ret = NETDEV_TX_BUSY;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100252
253 if (unlikely(!netif_running(dev) ||
Daniel Borkmann43279502014-03-27 16:38:30 +0100254 !netif_carrier_ok(dev)))
255 goto drop;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100256
257 features = netif_skb_features(skb);
258 if (skb_needs_linearize(skb, features) &&
Daniel Borkmann43279502014-03-27 16:38:30 +0100259 __skb_linearize(skb))
260 goto drop;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100261
Daniel Borkmann10c51b56232014-08-27 11:11:27 +0200262 txq = skb_get_tx_queue(dev, skb);
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100263
Daniel Borkmann43279502014-03-27 16:38:30 +0100264 local_bh_disable();
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100265
Daniel Borkmann43279502014-03-27 16:38:30 +0100266 HARD_TX_LOCK(dev, txq, smp_processor_id());
David S. Miller10b3ad82014-08-29 21:07:24 -0700267 if (!netif_xmit_frozen_or_drv_stopped(txq))
David S. Millerfa2dbdc2014-08-29 21:55:22 -0700268 ret = netdev_start_xmit(skb, dev, txq, false);
Daniel Borkmann43279502014-03-27 16:38:30 +0100269 HARD_TX_UNLOCK(dev, txq);
270
271 local_bh_enable();
272
273 if (!dev_xmit_complete(ret))
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100274 kfree_skb(skb);
Daniel Borkmann43279502014-03-27 16:38:30 +0100275
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100276 return ret;
Daniel Borkmann43279502014-03-27 16:38:30 +0100277drop:
Daniel Borkmann0f97ede2014-04-02 20:52:56 +0200278 atomic_long_inc(&dev->tx_dropped);
Daniel Borkmann43279502014-03-27 16:38:30 +0100279 kfree_skb(skb);
280 return NET_XMIT_DROP;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100281}
282
Daniel Borkmann66e56cd2013-12-06 11:36:15 +0100283static struct net_device *packet_cached_dev_get(struct packet_sock *po)
284{
285 struct net_device *dev;
286
287 rcu_read_lock();
288 dev = rcu_dereference(po->cached_dev);
289 if (likely(dev))
290 dev_hold(dev);
291 rcu_read_unlock();
292
293 return dev;
294}
295
296static void packet_cached_dev_assign(struct packet_sock *po,
297 struct net_device *dev)
298{
299 rcu_assign_pointer(po->cached_dev, dev);
300}
301
302static void packet_cached_dev_reset(struct packet_sock *po)
303{
304 RCU_INIT_POINTER(po->cached_dev, NULL);
305}
306
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100307static bool packet_use_direct_xmit(const struct packet_sock *po)
308{
309 return po->xmit == packet_direct_xmit;
310}
311
Daniel Borkmann0fd5d572014-02-16 15:55:22 +0100312static u16 __packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb)
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100313{
Li Zhong1cbac012013-12-12 22:39:55 +0100314 return (u16) raw_smp_processor_id() % dev->real_num_tx_queues;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100315}
316
Daniel Borkmann0fd5d572014-02-16 15:55:22 +0100317static void packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb)
318{
319 const struct net_device_ops *ops = dev->netdev_ops;
320 u16 queue_index;
321
322 if (ops->ndo_select_queue) {
323 queue_index = ops->ndo_select_queue(dev, skb, NULL,
324 __packet_pick_tx_queue);
325 queue_index = netdev_cap_txqueue(dev, queue_index);
326 } else {
327 queue_index = __packet_pick_tx_queue(dev, skb);
328 }
329
330 skb_set_queue_mapping(skb, queue_index);
331}
332
David S. Millerce06b032011-07-04 01:44:29 -0700333/* register_prot_hook must be invoked with the po->bind_lock held,
334 * or from a context in which asynchronous accesses to the packet
335 * socket is not possible (packet_create()).
336 */
337static void register_prot_hook(struct sock *sk)
338{
339 struct packet_sock *po = pkt_sk(sk);
Daniel Borkmanne40526c2013-11-21 16:50:58 +0100340
David S. Millerce06b032011-07-04 01:44:29 -0700341 if (!po->running) {
Daniel Borkmann66e56cd2013-12-06 11:36:15 +0100342 if (po->fanout)
David S. Millerdc99f602011-07-05 01:45:05 -0700343 __fanout_link(sk, po);
Daniel Borkmann66e56cd2013-12-06 11:36:15 +0100344 else
David S. Millerdc99f602011-07-05 01:45:05 -0700345 dev_add_pack(&po->prot_hook);
Daniel Borkmanne40526c2013-11-21 16:50:58 +0100346
David S. Millerce06b032011-07-04 01:44:29 -0700347 sock_hold(sk);
348 po->running = 1;
349 }
350}
351
352/* {,__}unregister_prot_hook() must be invoked with the po->bind_lock
353 * held. If the sync parameter is true, we will temporarily drop
354 * the po->bind_lock and do a synchronize_net to make sure no
355 * asynchronous packet processing paths still refer to the elements
356 * of po->prot_hook. If the sync parameter is false, it is the
357 * callers responsibility to take care of this.
358 */
359static void __unregister_prot_hook(struct sock *sk, bool sync)
360{
361 struct packet_sock *po = pkt_sk(sk);
362
363 po->running = 0;
Daniel Borkmann66e56cd2013-12-06 11:36:15 +0100364
365 if (po->fanout)
David S. Millerdc99f602011-07-05 01:45:05 -0700366 __fanout_unlink(sk, po);
Daniel Borkmann66e56cd2013-12-06 11:36:15 +0100367 else
David S. Millerdc99f602011-07-05 01:45:05 -0700368 __dev_remove_pack(&po->prot_hook);
Daniel Borkmanne40526c2013-11-21 16:50:58 +0100369
David S. Millerce06b032011-07-04 01:44:29 -0700370 __sock_put(sk);
371
372 if (sync) {
373 spin_unlock(&po->bind_lock);
374 synchronize_net();
375 spin_lock(&po->bind_lock);
376 }
377}
378
379static void unregister_prot_hook(struct sock *sk, bool sync)
380{
381 struct packet_sock *po = pkt_sk(sk);
382
383 if (po->running)
384 __unregister_prot_hook(sk, sync);
385}
386
Michael S. Tsirkin6e580402014-11-24 13:32:16 +0200387static inline struct page * __pure pgv_to_page(void *addr)
Changli Gao0af55bb2010-12-01 02:52:20 +0000388{
389 if (is_vmalloc_addr(addr))
390 return vmalloc_to_page(addr);
391 return virt_to_page(addr);
392}
393
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700394static void __packet_set_status(struct packet_sock *po, void *frame, int status)
395{
Daniel Borkmann184f4892013-04-16 01:57:46 +0000396 union tpacket_uhdr h;
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700397
398 h.raw = frame;
399 switch (po->tp_version) {
400 case TPACKET_V1:
401 h.h1->tp_status = status;
Changli Gao0af55bb2010-12-01 02:52:20 +0000402 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700403 break;
404 case TPACKET_V2:
405 h.h2->tp_status = status;
Changli Gao0af55bb2010-12-01 02:52:20 +0000406 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700407 break;
chetan lokef6fb8f102011-08-19 10:18:16 +0000408 case TPACKET_V3:
Johann Baudy69e3c752009-05-18 22:11:22 -0700409 default:
chetan lokef6fb8f102011-08-19 10:18:16 +0000410 WARN(1, "TPACKET version not supported.\n");
Johann Baudy69e3c752009-05-18 22:11:22 -0700411 BUG();
412 }
413
414 smp_wmb();
415}
416
417static int __packet_get_status(struct packet_sock *po, void *frame)
418{
Daniel Borkmann184f4892013-04-16 01:57:46 +0000419 union tpacket_uhdr h;
Johann Baudy69e3c752009-05-18 22:11:22 -0700420
421 smp_rmb();
422
423 h.raw = frame;
424 switch (po->tp_version) {
425 case TPACKET_V1:
Changli Gao0af55bb2010-12-01 02:52:20 +0000426 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
Johann Baudy69e3c752009-05-18 22:11:22 -0700427 return h.h1->tp_status;
428 case TPACKET_V2:
Changli Gao0af55bb2010-12-01 02:52:20 +0000429 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
Johann Baudy69e3c752009-05-18 22:11:22 -0700430 return h.h2->tp_status;
chetan lokef6fb8f102011-08-19 10:18:16 +0000431 case TPACKET_V3:
Johann Baudy69e3c752009-05-18 22:11:22 -0700432 default:
chetan lokef6fb8f102011-08-19 10:18:16 +0000433 WARN(1, "TPACKET version not supported.\n");
Johann Baudy69e3c752009-05-18 22:11:22 -0700434 BUG();
435 return 0;
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700436 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700437}
Johann Baudy69e3c752009-05-18 22:11:22 -0700438
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000439static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec *ts,
440 unsigned int flags)
Daniel Borkmann7a513842013-04-23 00:39:29 +0000441{
442 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
443
Willem de Bruijn68a360e2014-07-25 18:01:31 -0400444 if (shhwtstamps &&
445 (flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
446 ktime_to_timespec_cond(shhwtstamps->hwtstamp, ts))
447 return TP_STATUS_TS_RAW_HARDWARE;
Daniel Borkmann7a513842013-04-23 00:39:29 +0000448
449 if (ktime_to_timespec_cond(skb->tstamp, ts))
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000450 return TP_STATUS_TS_SOFTWARE;
Daniel Borkmann7a513842013-04-23 00:39:29 +0000451
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000452 return 0;
Daniel Borkmann7a513842013-04-23 00:39:29 +0000453}
454
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000455static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
456 struct sk_buff *skb)
Willem de Bruijn2e313962013-04-23 00:39:28 +0000457{
458 union tpacket_uhdr h;
459 struct timespec ts;
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000460 __u32 ts_status;
Willem de Bruijn2e313962013-04-23 00:39:28 +0000461
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000462 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
463 return 0;
Willem de Bruijn2e313962013-04-23 00:39:28 +0000464
465 h.raw = frame;
466 switch (po->tp_version) {
467 case TPACKET_V1:
468 h.h1->tp_sec = ts.tv_sec;
469 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
470 break;
471 case TPACKET_V2:
472 h.h2->tp_sec = ts.tv_sec;
473 h.h2->tp_nsec = ts.tv_nsec;
474 break;
475 case TPACKET_V3:
476 default:
477 WARN(1, "TPACKET version not supported.\n");
478 BUG();
479 }
480
481 /* one flush is safe, as both fields always lie on the same cacheline */
482 flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
483 smp_wmb();
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000484
485 return ts_status;
Willem de Bruijn2e313962013-04-23 00:39:28 +0000486}
487
Johann Baudy69e3c752009-05-18 22:11:22 -0700488static void *packet_lookup_frame(struct packet_sock *po,
489 struct packet_ring_buffer *rb,
490 unsigned int position,
491 int status)
492{
493 unsigned int pg_vec_pos, frame_offset;
Daniel Borkmann184f4892013-04-16 01:57:46 +0000494 union tpacket_uhdr h;
Johann Baudy69e3c752009-05-18 22:11:22 -0700495
496 pg_vec_pos = position / rb->frames_per_block;
497 frame_offset = position % rb->frames_per_block;
498
Neil Horman0e3125c2010-11-16 10:26:47 -0800499 h.raw = rb->pg_vec[pg_vec_pos].buffer +
500 (frame_offset * rb->frame_size);
Johann Baudy69e3c752009-05-18 22:11:22 -0700501
502 if (status != __packet_get_status(po, h.raw))
503 return NULL;
504
505 return h.raw;
506}
507
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000508static void *packet_current_frame(struct packet_sock *po,
Johann Baudy69e3c752009-05-18 22:11:22 -0700509 struct packet_ring_buffer *rb,
510 int status)
511{
512 return packet_lookup_frame(po, rb, rb->head, status);
513}
514
chetan lokebc59ba32011-08-25 10:43:30 +0000515static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
chetan lokef6fb8f102011-08-19 10:18:16 +0000516{
517 del_timer_sync(&pkc->retire_blk_timer);
518}
519
520static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
521 int tx_ring,
522 struct sk_buff_head *rb_queue)
523{
chetan lokebc59ba32011-08-25 10:43:30 +0000524 struct tpacket_kbdq_core *pkc;
chetan lokef6fb8f102011-08-19 10:18:16 +0000525
Duan Jiong22781a52013-12-06 13:29:36 +0800526 pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :
527 GET_PBDQC_FROM_RB(&po->rx_ring);
chetan lokef6fb8f102011-08-19 10:18:16 +0000528
Veaceslav Falicoec6f8092013-11-29 09:53:23 +0100529 spin_lock_bh(&rb_queue->lock);
chetan lokef6fb8f102011-08-19 10:18:16 +0000530 pkc->delete_blk_timer = 1;
Veaceslav Falicoec6f8092013-11-29 09:53:23 +0100531 spin_unlock_bh(&rb_queue->lock);
chetan lokef6fb8f102011-08-19 10:18:16 +0000532
533 prb_del_retire_blk_timer(pkc);
534}
535
536static void prb_init_blk_timer(struct packet_sock *po,
chetan lokebc59ba32011-08-25 10:43:30 +0000537 struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f102011-08-19 10:18:16 +0000538 void (*func) (unsigned long))
539{
540 init_timer(&pkc->retire_blk_timer);
541 pkc->retire_blk_timer.data = (long)po;
542 pkc->retire_blk_timer.function = func;
543 pkc->retire_blk_timer.expires = jiffies;
544}
545
546static void prb_setup_retire_blk_timer(struct packet_sock *po, int tx_ring)
547{
chetan lokebc59ba32011-08-25 10:43:30 +0000548 struct tpacket_kbdq_core *pkc;
chetan lokef6fb8f102011-08-19 10:18:16 +0000549
550 if (tx_ring)
551 BUG();
552
Duan Jiong22781a52013-12-06 13:29:36 +0800553 pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :
554 GET_PBDQC_FROM_RB(&po->rx_ring);
chetan lokef6fb8f102011-08-19 10:18:16 +0000555 prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);
556}
557
558static int prb_calc_retire_blk_tmo(struct packet_sock *po,
559 int blk_size_in_bytes)
560{
561 struct net_device *dev;
562 unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
Jiri Pirko4bc71cb2011-09-03 03:34:30 +0000563 struct ethtool_cmd ecmd;
564 int err;
parav.pandit@emulex.come440cf22012-06-27 03:56:12 +0000565 u32 speed;
chetan lokef6fb8f102011-08-19 10:18:16 +0000566
Jiri Pirko4bc71cb2011-09-03 03:34:30 +0000567 rtnl_lock();
568 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
569 if (unlikely(!dev)) {
570 rtnl_unlock();
chetan lokef6fb8f102011-08-19 10:18:16 +0000571 return DEFAULT_PRB_RETIRE_TOV;
Jiri Pirko4bc71cb2011-09-03 03:34:30 +0000572 }
573 err = __ethtool_get_settings(dev, &ecmd);
parav.pandit@emulex.come440cf22012-06-27 03:56:12 +0000574 speed = ethtool_cmd_speed(&ecmd);
Jiri Pirko4bc71cb2011-09-03 03:34:30 +0000575 rtnl_unlock();
576 if (!err) {
Jiri Pirko4bc71cb2011-09-03 03:34:30 +0000577 /*
578 * If the link speed is so slow you don't really
579 * need to worry about perf anyways
580 */
parav.pandit@emulex.come440cf22012-06-27 03:56:12 +0000581 if (speed < SPEED_1000 || speed == SPEED_UNKNOWN) {
Jiri Pirko4bc71cb2011-09-03 03:34:30 +0000582 return DEFAULT_PRB_RETIRE_TOV;
parav.pandit@emulex.come440cf22012-06-27 03:56:12 +0000583 } else {
584 msec = 1;
585 div = speed / 1000;
chetan lokef6fb8f102011-08-19 10:18:16 +0000586 }
587 }
588
589 mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
590
591 if (div)
592 mbits /= div;
593
594 tmo = mbits * msec;
595
596 if (div)
597 return tmo+1;
598 return tmo;
599}
600
chetan lokebc59ba32011-08-25 10:43:30 +0000601static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
chetan lokef6fb8f102011-08-19 10:18:16 +0000602 union tpacket_req_u *req_u)
603{
604 p1->feature_req_word = req_u->req3.tp_feature_req_word;
605}
606
607static void init_prb_bdqc(struct packet_sock *po,
608 struct packet_ring_buffer *rb,
609 struct pgv *pg_vec,
610 union tpacket_req_u *req_u, int tx_ring)
611{
Duan Jiong22781a52013-12-06 13:29:36 +0800612 struct tpacket_kbdq_core *p1 = GET_PBDQC_FROM_RB(rb);
chetan lokebc59ba32011-08-25 10:43:30 +0000613 struct tpacket_block_desc *pbd;
chetan lokef6fb8f102011-08-19 10:18:16 +0000614
615 memset(p1, 0x0, sizeof(*p1));
616
617 p1->knxt_seq_num = 1;
618 p1->pkbdq = pg_vec;
chetan lokebc59ba32011-08-25 10:43:30 +0000619 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
Joe Perchese3192692012-06-03 17:41:40 +0000620 p1->pkblk_start = pg_vec[0].buffer;
chetan lokef6fb8f102011-08-19 10:18:16 +0000621 p1->kblk_size = req_u->req3.tp_block_size;
622 p1->knum_blocks = req_u->req3.tp_block_nr;
623 p1->hdrlen = po->tp_hdrlen;
624 p1->version = po->tp_version;
625 p1->last_kactive_blk_num = 0;
Daniel Borkmannee80fbf2013-04-19 06:12:29 +0000626 po->stats.stats3.tp_freeze_q_cnt = 0;
chetan lokef6fb8f102011-08-19 10:18:16 +0000627 if (req_u->req3.tp_retire_blk_tov)
628 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
629 else
630 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
631 req_u->req3.tp_block_size);
632 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
633 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
634
Eric Dumazetdc808112014-08-15 09:16:04 -0700635 p1->max_frame_len = p1->kblk_size - BLK_PLUS_PRIV(p1->blk_sizeof_priv);
chetan lokef6fb8f102011-08-19 10:18:16 +0000636 prb_init_ft_ops(p1, req_u);
637 prb_setup_retire_blk_timer(po, tx_ring);
638 prb_open_block(p1, pbd);
639}
640
641/* Do NOT update the last_blk_num first.
642 * Assumes sk_buff_head lock is held.
643 */
chetan lokebc59ba32011-08-25 10:43:30 +0000644static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
chetan lokef6fb8f102011-08-19 10:18:16 +0000645{
646 mod_timer(&pkc->retire_blk_timer,
647 jiffies + pkc->tov_in_jiffies);
648 pkc->last_kactive_blk_num = pkc->kactive_blk_num;
649}
650
651/*
652 * Timer logic:
653 * 1) We refresh the timer only when we open a block.
654 * By doing this we don't waste cycles refreshing the timer
655 * on packet-by-packet basis.
656 *
657 * With a 1MB block-size, on a 1Gbps line, it will take
658 * i) ~8 ms to fill a block + ii) memcpy etc.
659 * In this cut we are not accounting for the memcpy time.
660 *
661 * So, if the user sets the 'tmo' to 10ms then the timer
662 * will never fire while the block is still getting filled
663 * (which is what we want). However, the user could choose
664 * to close a block early and that's fine.
665 *
666 * But when the timer does fire, we check whether or not to refresh it.
667 * Since the tmo granularity is in msecs, it is not too expensive
668 * to refresh the timer, lets say every '8' msecs.
669 * Either the user can set the 'tmo' or we can derive it based on
670 * a) line-speed and b) block-size.
671 * prb_calc_retire_blk_tmo() calculates the tmo.
672 *
673 */
674static void prb_retire_rx_blk_timer_expired(unsigned long data)
675{
676 struct packet_sock *po = (struct packet_sock *)data;
Duan Jiong22781a52013-12-06 13:29:36 +0800677 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
chetan lokef6fb8f102011-08-19 10:18:16 +0000678 unsigned int frozen;
chetan lokebc59ba32011-08-25 10:43:30 +0000679 struct tpacket_block_desc *pbd;
chetan lokef6fb8f102011-08-19 10:18:16 +0000680
681 spin_lock(&po->sk.sk_receive_queue.lock);
682
683 frozen = prb_queue_frozen(pkc);
684 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
685
686 if (unlikely(pkc->delete_blk_timer))
687 goto out;
688
689 /* We only need to plug the race when the block is partially filled.
690 * tpacket_rcv:
691 * lock(); increment BLOCK_NUM_PKTS; unlock()
692 * copy_bits() is in progress ...
693 * timer fires on other cpu:
694 * we can't retire the current block because copy_bits
695 * is in progress.
696 *
697 */
698 if (BLOCK_NUM_PKTS(pbd)) {
699 while (atomic_read(&pkc->blk_fill_in_prog)) {
700 /* Waiting for skb_copy_bits to finish... */
701 cpu_relax();
702 }
703 }
704
705 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
706 if (!frozen) {
Alexander Drozdov41a50d62015-02-24 08:18:28 +0300707 if (!BLOCK_NUM_PKTS(pbd)) {
708 /* An empty block. Just refresh the timer. */
709 goto refresh_timer;
710 }
chetan lokef6fb8f102011-08-19 10:18:16 +0000711 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
712 if (!prb_dispatch_next_block(pkc, po))
713 goto refresh_timer;
714 else
715 goto out;
716 } else {
717 /* Case 1. Queue was frozen because user-space was
718 * lagging behind.
719 */
720 if (prb_curr_blk_in_use(pkc, pbd)) {
721 /*
722 * Ok, user-space is still behind.
723 * So just refresh the timer.
724 */
725 goto refresh_timer;
726 } else {
727 /* Case 2. queue was frozen,user-space caught up,
728 * now the link went idle && the timer fired.
729 * We don't have a block to close.So we open this
730 * block and restart the timer.
731 * opening a block thaws the queue,restarts timer
732 * Thawing/timer-refresh is a side effect.
733 */
734 prb_open_block(pkc, pbd);
735 goto out;
736 }
737 }
738 }
739
740refresh_timer:
741 _prb_refresh_rx_retire_blk_timer(pkc);
742
743out:
744 spin_unlock(&po->sk.sk_receive_queue.lock);
745}
746
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000747static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
chetan lokebc59ba32011-08-25 10:43:30 +0000748 struct tpacket_block_desc *pbd1, __u32 status)
chetan lokef6fb8f102011-08-19 10:18:16 +0000749{
750 /* Flush everything minus the block header */
751
752#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
753 u8 *start, *end;
754
755 start = (u8 *)pbd1;
756
757 /* Skip the block header(we know header WILL fit in 4K) */
758 start += PAGE_SIZE;
759
760 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
761 for (; start < end; start += PAGE_SIZE)
762 flush_dcache_page(pgv_to_page(start));
763
764 smp_wmb();
765#endif
766
767 /* Now update the block status. */
768
769 BLOCK_STATUS(pbd1) = status;
770
771 /* Flush the block header */
772
773#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
774 start = (u8 *)pbd1;
775 flush_dcache_page(pgv_to_page(start));
776
777 smp_wmb();
778#endif
779}
780
781/*
782 * Side effect:
783 *
784 * 1) flush the block
785 * 2) Increment active_blk_num
786 *
787 * Note:We DONT refresh the timer on purpose.
788 * Because almost always the next block will be opened.
789 */
chetan lokebc59ba32011-08-25 10:43:30 +0000790static void prb_close_block(struct tpacket_kbdq_core *pkc1,
791 struct tpacket_block_desc *pbd1,
chetan lokef6fb8f102011-08-19 10:18:16 +0000792 struct packet_sock *po, unsigned int stat)
793{
794 __u32 status = TP_STATUS_USER | stat;
795
796 struct tpacket3_hdr *last_pkt;
chetan lokebc59ba32011-08-25 10:43:30 +0000797 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
Dan Collinsda413ee2014-12-19 16:49:25 +1300798 struct sock *sk = &po->sk;
chetan lokef6fb8f102011-08-19 10:18:16 +0000799
Daniel Borkmannee80fbf2013-04-19 06:12:29 +0000800 if (po->stats.stats3.tp_drops)
chetan lokef6fb8f102011-08-19 10:18:16 +0000801 status |= TP_STATUS_LOSING;
802
803 last_pkt = (struct tpacket3_hdr *)pkc1->prev;
804 last_pkt->tp_next_offset = 0;
805
806 /* Get the ts of the last pkt */
807 if (BLOCK_NUM_PKTS(pbd1)) {
808 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
809 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
810 } else {
Alexander Drozdov41a50d62015-02-24 08:18:28 +0300811 /* Ok, we tmo'd - so get the current time.
812 *
813 * It shouldn't really happen as we don't close empty
814 * blocks. See prb_retire_rx_blk_timer_expired().
815 */
chetan lokef6fb8f102011-08-19 10:18:16 +0000816 struct timespec ts;
817 getnstimeofday(&ts);
818 h1->ts_last_pkt.ts_sec = ts.tv_sec;
819 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
820 }
821
822 smp_wmb();
823
824 /* Flush the block */
825 prb_flush_block(pkc1, pbd1, status);
826
Dan Collinsda413ee2014-12-19 16:49:25 +1300827 sk->sk_data_ready(sk);
828
chetan lokef6fb8f102011-08-19 10:18:16 +0000829 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
830}
831
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000832static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
chetan lokef6fb8f102011-08-19 10:18:16 +0000833{
834 pkc->reset_pending_on_curr_blk = 0;
835}
836
837/*
838 * Side effect of opening a block:
839 *
840 * 1) prb_queue is thawed.
841 * 2) retire_blk_timer is refreshed.
842 *
843 */
chetan lokebc59ba32011-08-25 10:43:30 +0000844static void prb_open_block(struct tpacket_kbdq_core *pkc1,
845 struct tpacket_block_desc *pbd1)
chetan lokef6fb8f102011-08-19 10:18:16 +0000846{
847 struct timespec ts;
chetan lokebc59ba32011-08-25 10:43:30 +0000848 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
chetan lokef6fb8f102011-08-19 10:18:16 +0000849
850 smp_rmb();
851
Daniel Borkmann8da30562013-05-03 02:57:00 +0000852 /* We could have just memset this but we will lose the
853 * flexibility of making the priv area sticky
854 */
chetan lokef6fb8f102011-08-19 10:18:16 +0000855
Daniel Borkmann8da30562013-05-03 02:57:00 +0000856 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
857 BLOCK_NUM_PKTS(pbd1) = 0;
858 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
chetan lokef6fb8f102011-08-19 10:18:16 +0000859
Daniel Borkmann8da30562013-05-03 02:57:00 +0000860 getnstimeofday(&ts);
chetan lokef6fb8f102011-08-19 10:18:16 +0000861
Daniel Borkmann8da30562013-05-03 02:57:00 +0000862 h1->ts_first_pkt.ts_sec = ts.tv_sec;
863 h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
chetan lokef6fb8f102011-08-19 10:18:16 +0000864
Daniel Borkmann8da30562013-05-03 02:57:00 +0000865 pkc1->pkblk_start = (char *)pbd1;
866 pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
867
868 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
869 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
870
871 pbd1->version = pkc1->version;
872 pkc1->prev = pkc1->nxt_offset;
873 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
874
875 prb_thaw_queue(pkc1);
876 _prb_refresh_rx_retire_blk_timer(pkc1);
877
878 smp_wmb();
chetan lokef6fb8f102011-08-19 10:18:16 +0000879}
880
881/*
882 * Queue freeze logic:
883 * 1) Assume tp_block_nr = 8 blocks.
884 * 2) At time 't0', user opens Rx ring.
885 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
886 * 4) user-space is either sleeping or processing block '0'.
887 * 5) tpacket_rcv is currently filling block '7', since there is no space left,
888 * it will close block-7,loop around and try to fill block '0'.
889 * call-flow:
890 * __packet_lookup_frame_in_block
891 * prb_retire_current_block()
892 * prb_dispatch_next_block()
893 * |->(BLOCK_STATUS == USER) evaluates to true
894 * 5.1) Since block-0 is currently in-use, we just freeze the queue.
895 * 6) Now there are two cases:
896 * 6.1) Link goes idle right after the queue is frozen.
897 * But remember, the last open_block() refreshed the timer.
898 * When this timer expires,it will refresh itself so that we can
899 * re-open block-0 in near future.
900 * 6.2) Link is busy and keeps on receiving packets. This is a simple
901 * case and __packet_lookup_frame_in_block will check if block-0
902 * is free and can now be re-used.
903 */
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000904static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f102011-08-19 10:18:16 +0000905 struct packet_sock *po)
906{
907 pkc->reset_pending_on_curr_blk = 1;
Daniel Borkmannee80fbf2013-04-19 06:12:29 +0000908 po->stats.stats3.tp_freeze_q_cnt++;
chetan lokef6fb8f102011-08-19 10:18:16 +0000909}
910
911#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
912
913/*
914 * If the next block is free then we will dispatch it
915 * and return a good offset.
916 * Else, we will freeze the queue.
917 * So, caller must check the return value.
918 */
chetan lokebc59ba32011-08-25 10:43:30 +0000919static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f102011-08-19 10:18:16 +0000920 struct packet_sock *po)
921{
chetan lokebc59ba32011-08-25 10:43:30 +0000922 struct tpacket_block_desc *pbd;
chetan lokef6fb8f102011-08-19 10:18:16 +0000923
924 smp_rmb();
925
926 /* 1. Get current block num */
927 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
928
929 /* 2. If this block is currently in_use then freeze the queue */
930 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
931 prb_freeze_queue(pkc, po);
932 return NULL;
933 }
934
935 /*
936 * 3.
937 * open this block and return the offset where the first packet
938 * needs to get stored.
939 */
940 prb_open_block(pkc, pbd);
941 return (void *)pkc->nxt_offset;
942}
943
chetan lokebc59ba32011-08-25 10:43:30 +0000944static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f102011-08-19 10:18:16 +0000945 struct packet_sock *po, unsigned int status)
946{
chetan lokebc59ba32011-08-25 10:43:30 +0000947 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
chetan lokef6fb8f102011-08-19 10:18:16 +0000948
949 /* retire/close the current block */
950 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
951 /*
952 * Plug the case where copy_bits() is in progress on
953 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
954 * have space to copy the pkt in the current block and
955 * called prb_retire_current_block()
956 *
957 * We don't need to worry about the TMO case because
958 * the timer-handler already handled this case.
959 */
960 if (!(status & TP_STATUS_BLK_TMO)) {
961 while (atomic_read(&pkc->blk_fill_in_prog)) {
962 /* Waiting for skb_copy_bits to finish... */
963 cpu_relax();
964 }
965 }
966 prb_close_block(pkc, pbd, po, status);
967 return;
968 }
chetan lokef6fb8f102011-08-19 10:18:16 +0000969}
970
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000971static int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,
chetan lokebc59ba32011-08-25 10:43:30 +0000972 struct tpacket_block_desc *pbd)
chetan lokef6fb8f102011-08-19 10:18:16 +0000973{
974 return TP_STATUS_USER & BLOCK_STATUS(pbd);
975}
976
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000977static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
chetan lokef6fb8f102011-08-19 10:18:16 +0000978{
979 return pkc->reset_pending_on_curr_blk;
980}
981
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000982static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
chetan lokef6fb8f102011-08-19 10:18:16 +0000983{
chetan lokebc59ba32011-08-25 10:43:30 +0000984 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
chetan lokef6fb8f102011-08-19 10:18:16 +0000985 atomic_dec(&pkc->blk_fill_in_prog);
986}
987
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000988static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f102011-08-19 10:18:16 +0000989 struct tpacket3_hdr *ppd)
990{
Tom Herbert3958afa1b2013-12-15 22:12:06 -0800991 ppd->hv1.tp_rxhash = skb_get_hash(pkc->skb);
chetan lokef6fb8f102011-08-19 10:18:16 +0000992}
993
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000994static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f102011-08-19 10:18:16 +0000995 struct tpacket3_hdr *ppd)
996{
997 ppd->hv1.tp_rxhash = 0;
998}
999
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001000static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f102011-08-19 10:18:16 +00001001 struct tpacket3_hdr *ppd)
1002{
Jiri Pirkodf8a39d2015-01-13 17:13:44 +01001003 if (skb_vlan_tag_present(pkc->skb)) {
1004 ppd->hv1.tp_vlan_tci = skb_vlan_tag_get(pkc->skb);
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +09001005 ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->vlan_proto);
1006 ppd->tp_status = TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
chetan lokef6fb8f102011-08-19 10:18:16 +00001007 } else {
danborkmann@iogearbox.net9e670302012-08-20 03:34:03 +00001008 ppd->hv1.tp_vlan_tci = 0;
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +09001009 ppd->hv1.tp_vlan_tpid = 0;
danborkmann@iogearbox.net9e670302012-08-20 03:34:03 +00001010 ppd->tp_status = TP_STATUS_AVAILABLE;
chetan lokef6fb8f102011-08-19 10:18:16 +00001011 }
1012}
1013
chetan lokebc59ba32011-08-25 10:43:30 +00001014static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f102011-08-19 10:18:16 +00001015 struct tpacket3_hdr *ppd)
1016{
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +09001017 ppd->hv1.tp_padding = 0;
chetan lokef6fb8f102011-08-19 10:18:16 +00001018 prb_fill_vlan_info(pkc, ppd);
1019
1020 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
1021 prb_fill_rxhash(pkc, ppd);
1022 else
1023 prb_clear_rxhash(pkc, ppd);
1024}
1025
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001026static void prb_fill_curr_block(char *curr,
chetan lokebc59ba32011-08-25 10:43:30 +00001027 struct tpacket_kbdq_core *pkc,
1028 struct tpacket_block_desc *pbd,
chetan lokef6fb8f102011-08-19 10:18:16 +00001029 unsigned int len)
1030{
1031 struct tpacket3_hdr *ppd;
1032
1033 ppd = (struct tpacket3_hdr *)curr;
1034 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
1035 pkc->prev = curr;
1036 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
1037 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
1038 BLOCK_NUM_PKTS(pbd) += 1;
1039 atomic_inc(&pkc->blk_fill_in_prog);
1040 prb_run_all_ft_ops(pkc, ppd);
1041}
1042
1043/* Assumes caller has the sk->rx_queue.lock */
1044static void *__packet_lookup_frame_in_block(struct packet_sock *po,
1045 struct sk_buff *skb,
1046 int status,
1047 unsigned int len
1048 )
1049{
chetan lokebc59ba32011-08-25 10:43:30 +00001050 struct tpacket_kbdq_core *pkc;
1051 struct tpacket_block_desc *pbd;
chetan lokef6fb8f102011-08-19 10:18:16 +00001052 char *curr, *end;
1053
Joe Perchese3192692012-06-03 17:41:40 +00001054 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
chetan lokef6fb8f102011-08-19 10:18:16 +00001055 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1056
1057 /* Queue is frozen when user space is lagging behind */
1058 if (prb_queue_frozen(pkc)) {
1059 /*
1060 * Check if that last block which caused the queue to freeze,
1061 * is still in_use by user-space.
1062 */
1063 if (prb_curr_blk_in_use(pkc, pbd)) {
1064 /* Can't record this packet */
1065 return NULL;
1066 } else {
1067 /*
1068 * Ok, the block was released by user-space.
1069 * Now let's open that block.
1070 * opening a block also thaws the queue.
1071 * Thawing is a side effect.
1072 */
1073 prb_open_block(pkc, pbd);
1074 }
1075 }
1076
1077 smp_mb();
1078 curr = pkc->nxt_offset;
1079 pkc->skb = skb;
Joe Perchese3192692012-06-03 17:41:40 +00001080 end = (char *)pbd + pkc->kblk_size;
chetan lokef6fb8f102011-08-19 10:18:16 +00001081
1082 /* first try the current block */
1083 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
1084 prb_fill_curr_block(curr, pkc, pbd, len);
1085 return (void *)curr;
1086 }
1087
1088 /* Ok, close the current block */
1089 prb_retire_current_block(pkc, po, 0);
1090
1091 /* Now, try to dispatch the next block */
1092 curr = (char *)prb_dispatch_next_block(pkc, po);
1093 if (curr) {
1094 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1095 prb_fill_curr_block(curr, pkc, pbd, len);
1096 return (void *)curr;
1097 }
1098
1099 /*
1100 * No free blocks are available.user_space hasn't caught up yet.
1101 * Queue was just frozen and now this packet will get dropped.
1102 */
1103 return NULL;
1104}
1105
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001106static void *packet_current_rx_frame(struct packet_sock *po,
chetan lokef6fb8f102011-08-19 10:18:16 +00001107 struct sk_buff *skb,
1108 int status, unsigned int len)
1109{
1110 char *curr = NULL;
1111 switch (po->tp_version) {
1112 case TPACKET_V1:
1113 case TPACKET_V2:
1114 curr = packet_lookup_frame(po, &po->rx_ring,
1115 po->rx_ring.head, status);
1116 return curr;
1117 case TPACKET_V3:
1118 return __packet_lookup_frame_in_block(po, skb, status, len);
1119 default:
1120 WARN(1, "TPACKET version not supported\n");
1121 BUG();
Ying Xue99aa3472012-08-06 16:27:10 +00001122 return NULL;
chetan lokef6fb8f102011-08-19 10:18:16 +00001123 }
1124}
1125
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001126static void *prb_lookup_block(struct packet_sock *po,
chetan lokef6fb8f102011-08-19 10:18:16 +00001127 struct packet_ring_buffer *rb,
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001128 unsigned int idx,
chetan lokef6fb8f102011-08-19 10:18:16 +00001129 int status)
1130{
chetan lokebc59ba32011-08-25 10:43:30 +00001131 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001132 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
chetan lokef6fb8f102011-08-19 10:18:16 +00001133
1134 if (status != BLOCK_STATUS(pbd))
1135 return NULL;
1136 return pbd;
1137}
1138
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001139static int prb_previous_blk_num(struct packet_ring_buffer *rb)
chetan lokef6fb8f102011-08-19 10:18:16 +00001140{
1141 unsigned int prev;
1142 if (rb->prb_bdqc.kactive_blk_num)
1143 prev = rb->prb_bdqc.kactive_blk_num-1;
1144 else
1145 prev = rb->prb_bdqc.knum_blocks-1;
1146 return prev;
1147}
1148
1149/* Assumes caller has held the rx_queue.lock */
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001150static void *__prb_previous_block(struct packet_sock *po,
chetan lokef6fb8f102011-08-19 10:18:16 +00001151 struct packet_ring_buffer *rb,
1152 int status)
1153{
1154 unsigned int previous = prb_previous_blk_num(rb);
1155 return prb_lookup_block(po, rb, previous, status);
1156}
1157
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001158static void *packet_previous_rx_frame(struct packet_sock *po,
chetan lokef6fb8f102011-08-19 10:18:16 +00001159 struct packet_ring_buffer *rb,
1160 int status)
1161{
1162 if (po->tp_version <= TPACKET_V2)
1163 return packet_previous_frame(po, rb, status);
1164
1165 return __prb_previous_block(po, rb, status);
1166}
1167
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001168static void packet_increment_rx_head(struct packet_sock *po,
chetan lokef6fb8f102011-08-19 10:18:16 +00001169 struct packet_ring_buffer *rb)
1170{
1171 switch (po->tp_version) {
1172 case TPACKET_V1:
1173 case TPACKET_V2:
1174 return packet_increment_head(rb);
1175 case TPACKET_V3:
1176 default:
1177 WARN(1, "TPACKET version not supported.\n");
1178 BUG();
1179 return;
1180 }
1181}
1182
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001183static void *packet_previous_frame(struct packet_sock *po,
Johann Baudy69e3c752009-05-18 22:11:22 -07001184 struct packet_ring_buffer *rb,
1185 int status)
1186{
1187 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1188 return packet_lookup_frame(po, rb, previous, status);
1189}
1190
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001191static void packet_increment_head(struct packet_ring_buffer *buff)
Johann Baudy69e3c752009-05-18 22:11:22 -07001192{
1193 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1194}
1195
Daniel Borkmannb0138402014-01-15 16:25:36 +01001196static void packet_inc_pending(struct packet_ring_buffer *rb)
1197{
1198 this_cpu_inc(*rb->pending_refcnt);
1199}
1200
1201static void packet_dec_pending(struct packet_ring_buffer *rb)
1202{
1203 this_cpu_dec(*rb->pending_refcnt);
1204}
1205
1206static unsigned int packet_read_pending(const struct packet_ring_buffer *rb)
1207{
1208 unsigned int refcnt = 0;
1209 int cpu;
1210
1211 /* We don't use pending refcount in rx_ring. */
1212 if (rb->pending_refcnt == NULL)
1213 return 0;
1214
1215 for_each_possible_cpu(cpu)
1216 refcnt += *per_cpu_ptr(rb->pending_refcnt, cpu);
1217
1218 return refcnt;
1219}
1220
1221static int packet_alloc_pending(struct packet_sock *po)
1222{
1223 po->rx_ring.pending_refcnt = NULL;
1224
1225 po->tx_ring.pending_refcnt = alloc_percpu(unsigned int);
1226 if (unlikely(po->tx_ring.pending_refcnt == NULL))
1227 return -ENOBUFS;
1228
1229 return 0;
1230}
1231
1232static void packet_free_pending(struct packet_sock *po)
1233{
1234 free_percpu(po->tx_ring.pending_refcnt);
1235}
1236
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001237static bool packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1238{
1239 struct sock *sk = &po->sk;
1240 bool has_room;
1241
1242 if (po->prot_hook.func != tpacket_rcv)
1243 return (atomic_read(&sk->sk_rmem_alloc) + skb->truesize)
1244 <= sk->sk_rcvbuf;
1245
1246 spin_lock(&sk->sk_receive_queue.lock);
1247 if (po->tp_version == TPACKET_V3)
1248 has_room = prb_lookup_block(po, &po->rx_ring,
1249 po->rx_ring.prb_bdqc.kactive_blk_num,
1250 TP_STATUS_KERNEL);
1251 else
1252 has_room = packet_lookup_frame(po, &po->rx_ring,
1253 po->rx_ring.head,
1254 TP_STATUS_KERNEL);
1255 spin_unlock(&sk->sk_receive_queue.lock);
1256
1257 return has_room;
1258}
1259
Linus Torvalds1da177e2005-04-16 15:20:36 -07001260static void packet_sock_destruct(struct sock *sk)
1261{
Richard Cochraned85b562010-04-07 22:41:28 +00001262 skb_queue_purge(&sk->sk_error_queue);
1263
Ilpo Järvinen547b7922008-07-25 21:43:18 -07001264 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1265 WARN_ON(atomic_read(&sk->sk_wmem_alloc));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001266
1267 if (!sock_flag(sk, SOCK_DEAD)) {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001268 pr_err("Attempt to release alive packet socket: %p\n", sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001269 return;
1270 }
1271
Pavel Emelyanov17ab56a2007-11-10 21:38:48 -08001272 sk_refcnt_debug_dec(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001273}
1274
David S. Millerdc99f602011-07-05 01:45:05 -07001275static int fanout_rr_next(struct packet_fanout *f, unsigned int num)
1276{
1277 int x = atomic_read(&f->rr_cur) + 1;
1278
1279 if (x >= num)
1280 x = 0;
1281
1282 return x;
1283}
1284
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001285static unsigned int fanout_demux_hash(struct packet_fanout *f,
1286 struct sk_buff *skb,
1287 unsigned int num)
David S. Millerdc99f602011-07-05 01:45:05 -07001288{
Tom Herbert61b905d2014-03-24 15:34:47 -07001289 return reciprocal_scale(skb_get_hash(skb), num);
David S. Millerdc99f602011-07-05 01:45:05 -07001290}
1291
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001292static unsigned int fanout_demux_lb(struct packet_fanout *f,
1293 struct sk_buff *skb,
1294 unsigned int num)
David S. Millerdc99f602011-07-05 01:45:05 -07001295{
1296 int cur, old;
1297
1298 cur = atomic_read(&f->rr_cur);
1299 while ((old = atomic_cmpxchg(&f->rr_cur, cur,
1300 fanout_rr_next(f, num))) != cur)
1301 cur = old;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001302 return cur;
David S. Millerdc99f602011-07-05 01:45:05 -07001303}
1304
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001305static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1306 struct sk_buff *skb,
1307 unsigned int num)
David S. Miller95ec3eb2011-07-06 01:56:38 -07001308{
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001309 return smp_processor_id() % num;
1310}
David S. Miller95ec3eb2011-07-06 01:56:38 -07001311
Daniel Borkmann5df0ddf2013-08-28 22:13:09 +02001312static unsigned int fanout_demux_rnd(struct packet_fanout *f,
1313 struct sk_buff *skb,
1314 unsigned int num)
1315{
Daniel Borkmannf337db62014-01-22 02:29:39 +01001316 return prandom_u32_max(num);
Daniel Borkmann5df0ddf2013-08-28 22:13:09 +02001317}
1318
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001319static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1320 struct sk_buff *skb,
1321 unsigned int idx, unsigned int skip,
1322 unsigned int num)
1323{
1324 unsigned int i, j;
1325
1326 i = j = min_t(int, f->next[idx], num - 1);
1327 do {
1328 if (i != skip && packet_rcv_has_room(pkt_sk(f->arr[i]), skb)) {
1329 if (i != j)
1330 f->next[idx] = i;
1331 return i;
1332 }
1333 if (++i == num)
1334 i = 0;
1335 } while (i != j);
1336
1337 return idx;
1338}
1339
Neil Horman2d360972014-01-22 16:01:44 -05001340static unsigned int fanout_demux_qm(struct packet_fanout *f,
1341 struct sk_buff *skb,
1342 unsigned int num)
1343{
1344 return skb_get_queue_mapping(skb) % num;
1345}
1346
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001347static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1348{
1349 return f->flags & (flag >> 8);
David S. Miller95ec3eb2011-07-06 01:56:38 -07001350}
1351
David S. Miller95ec3eb2011-07-06 01:56:38 -07001352static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1353 struct packet_type *pt, struct net_device *orig_dev)
David S. Millerdc99f602011-07-05 01:45:05 -07001354{
1355 struct packet_fanout *f = pt->af_packet_priv;
1356 unsigned int num = f->num_members;
1357 struct packet_sock *po;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001358 unsigned int idx;
David S. Millerdc99f602011-07-05 01:45:05 -07001359
1360 if (!net_eq(dev_net(dev), read_pnet(&f->net)) ||
1361 !num) {
1362 kfree_skb(skb);
1363 return 0;
1364 }
1365
Alexander Drozdov3f34b242015-02-20 08:24:27 +03001366 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
1367 skb = ip_check_defrag(skb, IP_DEFRAG_AF_PACKET);
1368 if (!skb)
1369 return 0;
1370 }
David S. Miller95ec3eb2011-07-06 01:56:38 -07001371 switch (f->type) {
1372 case PACKET_FANOUT_HASH:
1373 default:
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001374 idx = fanout_demux_hash(f, skb, num);
David S. Miller95ec3eb2011-07-06 01:56:38 -07001375 break;
1376 case PACKET_FANOUT_LB:
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001377 idx = fanout_demux_lb(f, skb, num);
David S. Miller95ec3eb2011-07-06 01:56:38 -07001378 break;
1379 case PACKET_FANOUT_CPU:
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001380 idx = fanout_demux_cpu(f, skb, num);
1381 break;
Daniel Borkmann5df0ddf2013-08-28 22:13:09 +02001382 case PACKET_FANOUT_RND:
1383 idx = fanout_demux_rnd(f, skb, num);
1384 break;
Neil Horman2d360972014-01-22 16:01:44 -05001385 case PACKET_FANOUT_QM:
1386 idx = fanout_demux_qm(f, skb, num);
1387 break;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001388 case PACKET_FANOUT_ROLLOVER:
1389 idx = fanout_demux_rollover(f, skb, 0, (unsigned int) -1, num);
David S. Miller95ec3eb2011-07-06 01:56:38 -07001390 break;
David S. Miller7736d332011-07-05 01:43:20 -07001391 }
1392
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001393 po = pkt_sk(f->arr[idx]);
1394 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER) &&
1395 unlikely(!packet_rcv_has_room(po, skb))) {
1396 idx = fanout_demux_rollover(f, skb, idx, idx, num);
1397 po = pkt_sk(f->arr[idx]);
1398 }
David S. Millerdc99f602011-07-05 01:45:05 -07001399
1400 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1401}
1402
Pavel Emelyanovfff33212012-08-16 05:36:48 +00001403DEFINE_MUTEX(fanout_mutex);
1404EXPORT_SYMBOL_GPL(fanout_mutex);
David S. Millerdc99f602011-07-05 01:45:05 -07001405static LIST_HEAD(fanout_list);
1406
1407static void __fanout_link(struct sock *sk, struct packet_sock *po)
1408{
1409 struct packet_fanout *f = po->fanout;
1410
1411 spin_lock(&f->lock);
1412 f->arr[f->num_members] = sk;
1413 smp_wmb();
1414 f->num_members++;
1415 spin_unlock(&f->lock);
1416}
1417
1418static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1419{
1420 struct packet_fanout *f = po->fanout;
1421 int i;
1422
1423 spin_lock(&f->lock);
1424 for (i = 0; i < f->num_members; i++) {
1425 if (f->arr[i] == sk)
1426 break;
1427 }
1428 BUG_ON(i >= f->num_members);
1429 f->arr[i] = f->arr[f->num_members - 1];
1430 f->num_members--;
1431 spin_unlock(&f->lock);
1432}
1433
Weilong Chend4dd8ae2013-12-23 11:31:38 +08001434static bool match_fanout_group(struct packet_type *ptype, struct sock *sk)
Eric Leblondc0de08d2012-08-16 22:02:58 +00001435{
Weilong Chend4dd8ae2013-12-23 11:31:38 +08001436 if (ptype->af_packet_priv == (void *)((struct packet_sock *)sk)->fanout)
Eric Leblondc0de08d2012-08-16 22:02:58 +00001437 return true;
1438
1439 return false;
1440}
1441
David S. Miller7736d332011-07-05 01:43:20 -07001442static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
David S. Millerdc99f602011-07-05 01:45:05 -07001443{
1444 struct packet_sock *po = pkt_sk(sk);
1445 struct packet_fanout *f, *match;
David S. Miller7736d332011-07-05 01:43:20 -07001446 u8 type = type_flags & 0xff;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001447 u8 flags = type_flags >> 8;
David S. Millerdc99f602011-07-05 01:45:05 -07001448 int err;
1449
1450 switch (type) {
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001451 case PACKET_FANOUT_ROLLOVER:
1452 if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1453 return -EINVAL;
David S. Millerdc99f602011-07-05 01:45:05 -07001454 case PACKET_FANOUT_HASH:
1455 case PACKET_FANOUT_LB:
David S. Miller95ec3eb2011-07-06 01:56:38 -07001456 case PACKET_FANOUT_CPU:
Daniel Borkmann5df0ddf2013-08-28 22:13:09 +02001457 case PACKET_FANOUT_RND:
Neil Horman2d360972014-01-22 16:01:44 -05001458 case PACKET_FANOUT_QM:
David S. Millerdc99f602011-07-05 01:45:05 -07001459 break;
1460 default:
1461 return -EINVAL;
1462 }
1463
1464 if (!po->running)
1465 return -EINVAL;
1466
1467 if (po->fanout)
1468 return -EALREADY;
1469
1470 mutex_lock(&fanout_mutex);
1471 match = NULL;
1472 list_for_each_entry(f, &fanout_list, list) {
1473 if (f->id == id &&
1474 read_pnet(&f->net) == sock_net(sk)) {
1475 match = f;
1476 break;
1477 }
1478 }
Eric Dumazetafe62c62011-07-07 06:41:29 -07001479 err = -EINVAL;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001480 if (match && match->flags != flags)
Eric Dumazetafe62c62011-07-07 06:41:29 -07001481 goto out;
David S. Millerdc99f602011-07-05 01:45:05 -07001482 if (!match) {
Eric Dumazetafe62c62011-07-07 06:41:29 -07001483 err = -ENOMEM;
David S. Millerdc99f602011-07-05 01:45:05 -07001484 match = kzalloc(sizeof(*match), GFP_KERNEL);
Eric Dumazetafe62c62011-07-07 06:41:29 -07001485 if (!match)
1486 goto out;
1487 write_pnet(&match->net, sock_net(sk));
1488 match->id = id;
1489 match->type = type;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001490 match->flags = flags;
Eric Dumazetafe62c62011-07-07 06:41:29 -07001491 atomic_set(&match->rr_cur, 0);
1492 INIT_LIST_HEAD(&match->list);
1493 spin_lock_init(&match->lock);
1494 atomic_set(&match->sk_ref, 0);
1495 match->prot_hook.type = po->prot_hook.type;
1496 match->prot_hook.dev = po->prot_hook.dev;
1497 match->prot_hook.func = packet_rcv_fanout;
1498 match->prot_hook.af_packet_priv = match;
Eric Leblondc0de08d2012-08-16 22:02:58 +00001499 match->prot_hook.id_match = match_fanout_group;
Eric Dumazetafe62c62011-07-07 06:41:29 -07001500 dev_add_pack(&match->prot_hook);
1501 list_add(&match->list, &fanout_list);
1502 }
1503 err = -EINVAL;
1504 if (match->type == type &&
1505 match->prot_hook.type == po->prot_hook.type &&
1506 match->prot_hook.dev == po->prot_hook.dev) {
1507 err = -ENOSPC;
1508 if (atomic_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
1509 __dev_remove_pack(&po->prot_hook);
1510 po->fanout = match;
1511 atomic_inc(&match->sk_ref);
1512 __fanout_link(sk, po);
1513 err = 0;
David S. Millerdc99f602011-07-05 01:45:05 -07001514 }
1515 }
Eric Dumazetafe62c62011-07-07 06:41:29 -07001516out:
David S. Millerdc99f602011-07-05 01:45:05 -07001517 mutex_unlock(&fanout_mutex);
1518 return err;
1519}
1520
1521static void fanout_release(struct sock *sk)
1522{
1523 struct packet_sock *po = pkt_sk(sk);
1524 struct packet_fanout *f;
1525
1526 f = po->fanout;
1527 if (!f)
1528 return;
1529
Pavel Emelyanovfff33212012-08-16 05:36:48 +00001530 mutex_lock(&fanout_mutex);
David S. Millerdc99f602011-07-05 01:45:05 -07001531 po->fanout = NULL;
1532
David S. Millerdc99f602011-07-05 01:45:05 -07001533 if (atomic_dec_and_test(&f->sk_ref)) {
1534 list_del(&f->list);
1535 dev_remove_pack(&f->prot_hook);
1536 kfree(f);
1537 }
1538 mutex_unlock(&fanout_mutex);
1539}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001540
Eric Dumazet90ddc4f2005-12-22 12:49:22 -08001541static const struct proto_ops packet_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001542
Eric Dumazet90ddc4f2005-12-22 12:49:22 -08001543static const struct proto_ops packet_ops_spkt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001544
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001545static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1546 struct packet_type *pt, struct net_device *orig_dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001547{
1548 struct sock *sk;
1549 struct sockaddr_pkt *spkt;
1550
1551 /*
1552 * When we registered the protocol we saved the socket in the data
1553 * field for just this event.
1554 */
1555
1556 sk = pt->af_packet_priv;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001557
Linus Torvalds1da177e2005-04-16 15:20:36 -07001558 /*
1559 * Yank back the headers [hope the device set this
1560 * right or kerboom...]
1561 *
1562 * Incoming packets have ll header pulled,
1563 * push it back.
1564 *
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -07001565 * For outgoing ones skb->data == skb_mac_header(skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001566 * so that this procedure is noop.
1567 */
1568
1569 if (skb->pkt_type == PACKET_LOOPBACK)
1570 goto out;
1571
Octavian Purdila09ad9bc2009-11-25 15:14:13 -08001572 if (!net_eq(dev_net(dev), sock_net(sk)))
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08001573 goto out;
1574
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001575 skb = skb_share_check(skb, GFP_ATOMIC);
1576 if (skb == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001577 goto oom;
1578
1579 /* drop any routing info */
Eric Dumazetadf30902009-06-02 05:19:30 +00001580 skb_dst_drop(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001581
Phil Oester84531c22005-07-12 11:57:52 -07001582 /* drop conntrack reference */
1583 nf_reset(skb);
1584
Herbert Xuffbc6112007-02-04 23:33:10 -08001585 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001586
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -07001587 skb_push(skb, skb->data - skb_mac_header(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001588
1589 /*
1590 * The SOCK_PACKET socket receives _all_ frames.
1591 */
1592
1593 spkt->spkt_family = dev->type;
1594 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1595 spkt->spkt_protocol = skb->protocol;
1596
1597 /*
1598 * Charge the memory to the socket. This is done specifically
1599 * to prevent sockets using all the memory up.
1600 */
1601
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001602 if (sock_queue_rcv_skb(sk, skb) == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001603 return 0;
1604
1605out:
1606 kfree_skb(skb);
1607oom:
1608 return 0;
1609}
1610
1611
1612/*
1613 * Output a raw packet to a device layer. This bypasses all the other
1614 * protocol layers and you must therefore supply it with a complete frame
1615 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001616
Ying Xue1b784142015-03-02 15:37:48 +08001617static int packet_sendmsg_spkt(struct socket *sock, struct msghdr *msg,
1618 size_t len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001619{
1620 struct sock *sk = sock->sk;
Steffen Hurrle342dfc32014-01-17 22:53:15 +01001621 DECLARE_SOCKADDR(struct sockaddr_pkt *, saddr, msg->msg_name);
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001622 struct sk_buff *skb = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001623 struct net_device *dev;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001624 __be16 proto = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001625 int err;
Ben Greear3bdc0eb2012-02-11 15:39:30 +00001626 int extra_len = 0;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001627
Linus Torvalds1da177e2005-04-16 15:20:36 -07001628 /*
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001629 * Get and verify the address.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001630 */
1631
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001632 if (saddr) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001633 if (msg->msg_namelen < sizeof(struct sockaddr))
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001634 return -EINVAL;
1635 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1636 proto = saddr->spkt_protocol;
1637 } else
1638 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001639
1640 /*
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001641 * Find the device first to size check it
Linus Torvalds1da177e2005-04-16 15:20:36 -07001642 */
1643
danborkmann@iogearbox.netde74e922012-06-10 08:59:28 +00001644 saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001645retry:
Eric Dumazet654d1f82009-11-02 10:43:32 +01001646 rcu_read_lock();
1647 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001648 err = -ENODEV;
1649 if (dev == NULL)
1650 goto out_unlock;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001651
David S. Millerd5e76b02007-01-25 19:30:36 -08001652 err = -ENETDOWN;
1653 if (!(dev->flags & IFF_UP))
1654 goto out_unlock;
1655
Linus Torvalds1da177e2005-04-16 15:20:36 -07001656 /*
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001657 * You may not queue a frame bigger than the mtu. This is the lowest level
1658 * raw protocol and you must do your own fragmentation at this level.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001659 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001660
Ben Greear3bdc0eb2012-02-11 15:39:30 +00001661 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1662 if (!netif_supports_nofcs(dev)) {
1663 err = -EPROTONOSUPPORT;
1664 goto out_unlock;
1665 }
1666 extra_len = 4; /* We're doing our own CRC */
1667 }
1668
Linus Torvalds1da177e2005-04-16 15:20:36 -07001669 err = -EMSGSIZE;
Ben Greear3bdc0eb2012-02-11 15:39:30 +00001670 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001671 goto out_unlock;
1672
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001673 if (!skb) {
1674 size_t reserved = LL_RESERVED_SPACE(dev);
Herbert Xu4ce40912011-11-18 02:20:05 +00001675 int tlen = dev->needed_tailroom;
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001676 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001677
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001678 rcu_read_unlock();
Herbert Xu4ce40912011-11-18 02:20:05 +00001679 skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001680 if (skb == NULL)
1681 return -ENOBUFS;
1682 /* FIXME: Save some space for broken drivers that write a hard
1683 * header at transmission time by themselves. PPP is the notable
1684 * one here. This should really be fixed at the driver level.
1685 */
1686 skb_reserve(skb, reserved);
1687 skb_reset_network_header(skb);
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001688
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001689 /* Try to align data part correctly */
1690 if (hhlen) {
1691 skb->data -= hhlen;
1692 skb->tail -= hhlen;
1693 if (len < hhlen)
1694 skb_reset_network_header(skb);
1695 }
Al Viro6ce8e9c2014-04-06 21:25:44 -04001696 err = memcpy_from_msg(skb_put(skb, len), msg, len);
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001697 if (err)
1698 goto out_free;
1699 goto retry;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001700 }
1701
Ben Greear3bdc0eb2012-02-11 15:39:30 +00001702 if (len > (dev->mtu + dev->hard_header_len + extra_len)) {
Ben Greear57f89bf2011-02-11 09:35:18 +00001703 /* Earlier code assumed this would be a VLAN pkt,
1704 * double-check this now that we have the actual
1705 * packet in hand.
1706 */
1707 struct ethhdr *ehdr;
1708 skb_reset_mac_header(skb);
1709 ehdr = eth_hdr(skb);
1710 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
1711 err = -EMSGSIZE;
1712 goto out_unlock;
1713 }
1714 }
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001715
Linus Torvalds1da177e2005-04-16 15:20:36 -07001716 skb->protocol = proto;
1717 skb->dev = dev;
1718 skb->priority = sk->sk_priority;
Eric Dumazet2d37a182009-10-01 19:14:46 +00001719 skb->mark = sk->sk_mark;
Daniel Borkmannbf84a012013-04-14 08:08:13 +00001720
1721 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001722
Ben Greear3bdc0eb2012-02-11 15:39:30 +00001723 if (unlikely(extra_len == 4))
1724 skb->no_fcs = 1;
1725
Jason Wang40893fd2013-03-26 23:11:22 +00001726 skb_probe_transport_header(skb, 0);
Jason Wangc1aad272013-03-25 20:19:57 +00001727
Linus Torvalds1da177e2005-04-16 15:20:36 -07001728 dev_queue_xmit(skb);
Eric Dumazet654d1f82009-11-02 10:43:32 +01001729 rcu_read_unlock();
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001730 return len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001731
Linus Torvalds1da177e2005-04-16 15:20:36 -07001732out_unlock:
Eric Dumazet654d1f82009-11-02 10:43:32 +01001733 rcu_read_unlock();
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001734out_free:
1735 kfree_skb(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001736 return err;
1737}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001738
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001739static unsigned int run_filter(const struct sk_buff *skb,
Eric Dumazet62ab0812010-12-06 20:50:09 +00001740 const struct sock *sk,
David S. Millerdbcb5852007-01-24 15:21:02 -08001741 unsigned int res)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001742{
1743 struct sk_filter *filter;
1744
Eric Dumazet80f8f102011-01-18 07:46:52 +00001745 rcu_read_lock();
1746 filter = rcu_dereference(sk->sk_filter);
David S. Millerdbcb5852007-01-24 15:21:02 -08001747 if (filter != NULL)
Eric Dumazet0a148422011-04-20 09:27:32 +00001748 res = SK_RUN_FILTER(filter, skb);
Eric Dumazet80f8f102011-01-18 07:46:52 +00001749 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001750
David S. Millerdbcb5852007-01-24 15:21:02 -08001751 return res;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001752}
1753
1754/*
Eric Dumazet62ab0812010-12-06 20:50:09 +00001755 * This function makes lazy skb cloning in hope that most of packets
1756 * are discarded by BPF.
1757 *
1758 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
1759 * and skb->cb are mangled. It works because (and until) packets
1760 * falling here are owned by current CPU. Output packets are cloned
1761 * by dev_queue_xmit_nit(), input packets are processed by net_bh
1762 * sequencially, so that if we return skb to original state on exit,
1763 * we will not harm anyone.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001764 */
1765
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001766static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
1767 struct packet_type *pt, struct net_device *orig_dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001768{
1769 struct sock *sk;
1770 struct sockaddr_ll *sll;
1771 struct packet_sock *po;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001772 u8 *skb_head = skb->data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001773 int skb_len = skb->len;
David S. Millerdbcb5852007-01-24 15:21:02 -08001774 unsigned int snaplen, res;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001775
1776 if (skb->pkt_type == PACKET_LOOPBACK)
1777 goto drop;
1778
1779 sk = pt->af_packet_priv;
1780 po = pkt_sk(sk);
1781
Octavian Purdila09ad9bc2009-11-25 15:14:13 -08001782 if (!net_eq(dev_net(dev), sock_net(sk)))
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08001783 goto drop;
1784
Linus Torvalds1da177e2005-04-16 15:20:36 -07001785 skb->dev = dev;
1786
Stephen Hemminger3b04ddd2007-10-09 01:40:57 -07001787 if (dev->header_ops) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001788 /* The device has an explicit notion of ll header,
Eric Dumazet62ab0812010-12-06 20:50:09 +00001789 * exported to higher levels.
1790 *
1791 * Otherwise, the device hides details of its frame
1792 * structure, so that corresponding packet head is
1793 * never delivered to user.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001794 */
1795 if (sk->sk_type != SOCK_DGRAM)
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -07001796 skb_push(skb, skb->data - skb_mac_header(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001797 else if (skb->pkt_type == PACKET_OUTGOING) {
1798 /* Special case: outgoing packets have ll header at head */
Arnaldo Carvalho de Melobbe735e2007-03-10 22:16:10 -03001799 skb_pull(skb, skb_network_offset(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001800 }
1801 }
1802
1803 snaplen = skb->len;
1804
David S. Millerdbcb5852007-01-24 15:21:02 -08001805 res = run_filter(skb, sk, snaplen);
1806 if (!res)
Dmitry Mishinfda9ef52006-08-31 15:28:39 -07001807 goto drop_n_restore;
David S. Millerdbcb5852007-01-24 15:21:02 -08001808 if (snaplen > res)
1809 snaplen = res;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001810
Eric Dumazet0fd7bac2011-12-21 07:11:44 +00001811 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001812 goto drop_n_acct;
1813
1814 if (skb_shared(skb)) {
1815 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
1816 if (nskb == NULL)
1817 goto drop_n_acct;
1818
1819 if (skb_head != skb->data) {
1820 skb->data = skb_head;
1821 skb->len = skb_len;
1822 }
Eric Dumazetabc4e4f2012-04-19 02:24:42 +00001823 consume_skb(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001824 skb = nskb;
1825 }
1826
Eyal Birgerb4772ef2015-03-01 14:58:29 +02001827 sock_skb_cb_check_size(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8);
Herbert Xuffbc6112007-02-04 23:33:10 -08001828
1829 sll = &PACKET_SKB_CB(skb)->sa.ll;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001830 sll->sll_hatype = dev->type;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001831 sll->sll_pkttype = skb->pkt_type;
Peter P Waskiewicz Jr8032b462007-11-10 22:03:25 -08001832 if (unlikely(po->origdev))
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07001833 sll->sll_ifindex = orig_dev->ifindex;
1834 else
1835 sll->sll_ifindex = dev->ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001836
Stephen Hemmingerb95cce32007-09-26 22:13:38 -07001837 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001838
Eyal Birger2472d762015-03-01 14:58:28 +02001839 /* sll->sll_family and sll->sll_protocol are set in packet_recvmsg().
1840 * Use their space for storing the original skb length.
1841 */
1842 PACKET_SKB_CB(skb)->sa.origlen = skb->len;
Herbert Xu8dc41942007-02-04 23:31:32 -08001843
Linus Torvalds1da177e2005-04-16 15:20:36 -07001844 if (pskb_trim(skb, snaplen))
1845 goto drop_n_acct;
1846
1847 skb_set_owner_r(skb, sk);
1848 skb->dev = NULL;
Eric Dumazetadf30902009-06-02 05:19:30 +00001849 skb_dst_drop(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001850
Phil Oester84531c22005-07-12 11:57:52 -07001851 /* drop conntrack reference */
1852 nf_reset(skb);
1853
Linus Torvalds1da177e2005-04-16 15:20:36 -07001854 spin_lock(&sk->sk_receive_queue.lock);
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00001855 po->stats.stats1.tp_packets++;
Eyal Birger3bc3b962015-03-01 14:58:30 +02001856 sock_skb_set_dropcount(sk, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001857 __skb_queue_tail(&sk->sk_receive_queue, skb);
1858 spin_unlock(&sk->sk_receive_queue.lock);
David S. Miller676d2362014-04-11 16:15:36 -04001859 sk->sk_data_ready(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001860 return 0;
1861
1862drop_n_acct:
Willem de Bruijn7091fbd2011-09-30 10:38:28 +00001863 spin_lock(&sk->sk_receive_queue.lock);
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00001864 po->stats.stats1.tp_drops++;
Willem de Bruijn7091fbd2011-09-30 10:38:28 +00001865 atomic_inc(&sk->sk_drops);
1866 spin_unlock(&sk->sk_receive_queue.lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001867
1868drop_n_restore:
1869 if (skb_head != skb->data && skb_shared(skb)) {
1870 skb->data = skb_head;
1871 skb->len = skb_len;
1872 }
1873drop:
Neil Hormanead2ceb2009-03-11 09:49:55 +00001874 consume_skb(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001875 return 0;
1876}
1877
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001878static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
1879 struct packet_type *pt, struct net_device *orig_dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001880{
1881 struct sock *sk;
1882 struct packet_sock *po;
1883 struct sockaddr_ll *sll;
Daniel Borkmann184f4892013-04-16 01:57:46 +00001884 union tpacket_uhdr h;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001885 u8 *skb_head = skb->data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001886 int skb_len = skb->len;
David S. Millerdbcb5852007-01-24 15:21:02 -08001887 unsigned int snaplen, res;
chetan lokef6fb8f102011-08-19 10:18:16 +00001888 unsigned long status = TP_STATUS_USER;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07001889 unsigned short macoff, netoff, hdrlen;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001890 struct sk_buff *copy_skb = NULL;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07001891 struct timespec ts;
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +00001892 __u32 ts_status;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001893
Atzm Watanabe51846352013-12-17 22:53:32 +09001894 /* struct tpacket{2,3}_hdr is aligned to a multiple of TPACKET_ALIGNMENT.
1895 * We may add members to them until current aligned size without forcing
1896 * userspace to call getsockopt(..., PACKET_HDRLEN, ...).
1897 */
1898 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h2)) != 32);
1899 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h3)) != 48);
1900
Linus Torvalds1da177e2005-04-16 15:20:36 -07001901 if (skb->pkt_type == PACKET_LOOPBACK)
1902 goto drop;
1903
1904 sk = pt->af_packet_priv;
1905 po = pkt_sk(sk);
1906
Octavian Purdila09ad9bc2009-11-25 15:14:13 -08001907 if (!net_eq(dev_net(dev), sock_net(sk)))
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08001908 goto drop;
1909
Stephen Hemminger3b04ddd2007-10-09 01:40:57 -07001910 if (dev->header_ops) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001911 if (sk->sk_type != SOCK_DGRAM)
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -07001912 skb_push(skb, skb->data - skb_mac_header(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001913 else if (skb->pkt_type == PACKET_OUTGOING) {
1914 /* Special case: outgoing packets have ll header at head */
Arnaldo Carvalho de Melobbe735e2007-03-10 22:16:10 -03001915 skb_pull(skb, skb_network_offset(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001916 }
1917 }
1918
1919 snaplen = skb->len;
1920
David S. Millerdbcb5852007-01-24 15:21:02 -08001921 res = run_filter(skb, sk, snaplen);
1922 if (!res)
Dmitry Mishinfda9ef52006-08-31 15:28:39 -07001923 goto drop_n_restore;
Alexander Drozdov68c2e5d2015-03-23 09:11:12 +03001924
1925 if (skb->ip_summed == CHECKSUM_PARTIAL)
1926 status |= TP_STATUS_CSUMNOTREADY;
Alexander Drozdov682f0482015-03-23 09:11:13 +03001927 else if (skb->pkt_type != PACKET_OUTGOING &&
1928 (skb->ip_summed == CHECKSUM_COMPLETE ||
1929 skb_csum_unnecessary(skb)))
1930 status |= TP_STATUS_CSUM_VALID;
Alexander Drozdov68c2e5d2015-03-23 09:11:12 +03001931
David S. Millerdbcb5852007-01-24 15:21:02 -08001932 if (snaplen > res)
1933 snaplen = res;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001934
1935 if (sk->sk_type == SOCK_DGRAM) {
Patrick McHardy8913336a2008-07-18 18:05:19 -07001936 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
1937 po->tp_reserve;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001938 } else {
Eric Dumazet95c96172012-04-15 05:58:06 +00001939 unsigned int maclen = skb_network_offset(skb);
Patrick McHardybbd6ef82008-07-14 22:50:15 -07001940 netoff = TPACKET_ALIGN(po->tp_hdrlen +
Patrick McHardy8913336a2008-07-18 18:05:19 -07001941 (maclen < 16 ? 16 : maclen)) +
1942 po->tp_reserve;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001943 macoff = netoff - maclen;
1944 }
chetan lokef6fb8f102011-08-19 10:18:16 +00001945 if (po->tp_version <= TPACKET_V2) {
1946 if (macoff + snaplen > po->rx_ring.frame_size) {
1947 if (po->copy_thresh &&
Eric Dumazet0fd7bac2011-12-21 07:11:44 +00001948 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
chetan lokef6fb8f102011-08-19 10:18:16 +00001949 if (skb_shared(skb)) {
1950 copy_skb = skb_clone(skb, GFP_ATOMIC);
1951 } else {
1952 copy_skb = skb_get(skb);
1953 skb_head = skb->data;
1954 }
1955 if (copy_skb)
1956 skb_set_owner_r(copy_skb, sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001957 }
chetan lokef6fb8f102011-08-19 10:18:16 +00001958 snaplen = po->rx_ring.frame_size - macoff;
1959 if ((int)snaplen < 0)
1960 snaplen = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001961 }
Eric Dumazetdc808112014-08-15 09:16:04 -07001962 } else if (unlikely(macoff + snaplen >
1963 GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len)) {
1964 u32 nval;
1965
1966 nval = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len - macoff;
1967 pr_err_once("tpacket_rcv: packet too big, clamped from %u to %u. macoff=%u\n",
1968 snaplen, nval, macoff);
1969 snaplen = nval;
1970 if (unlikely((int)snaplen < 0)) {
1971 snaplen = 0;
1972 macoff = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len;
1973 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001974 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001975 spin_lock(&sk->sk_receive_queue.lock);
chetan lokef6fb8f102011-08-19 10:18:16 +00001976 h.raw = packet_current_rx_frame(po, skb,
1977 TP_STATUS_KERNEL, (macoff+snaplen));
Patrick McHardybbd6ef82008-07-14 22:50:15 -07001978 if (!h.raw)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001979 goto ring_is_full;
chetan lokef6fb8f102011-08-19 10:18:16 +00001980 if (po->tp_version <= TPACKET_V2) {
1981 packet_increment_rx_head(po, &po->rx_ring);
1982 /*
1983 * LOSING will be reported till you read the stats,
1984 * because it's COR - Clear On Read.
1985 * Anyways, moving it for V1/V2 only as V3 doesn't need this
1986 * at packet level.
1987 */
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00001988 if (po->stats.stats1.tp_drops)
chetan lokef6fb8f102011-08-19 10:18:16 +00001989 status |= TP_STATUS_LOSING;
1990 }
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00001991 po->stats.stats1.tp_packets++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001992 if (copy_skb) {
1993 status |= TP_STATUS_COPY;
1994 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
1995 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001996 spin_unlock(&sk->sk_receive_queue.lock);
1997
Patrick McHardybbd6ef82008-07-14 22:50:15 -07001998 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +00001999
2000 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
Daniel Borkmann7a513842013-04-23 00:39:29 +00002001 getnstimeofday(&ts);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002002
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +00002003 status |= ts_status;
2004
Patrick McHardybbd6ef82008-07-14 22:50:15 -07002005 switch (po->tp_version) {
2006 case TPACKET_V1:
2007 h.h1->tp_len = skb->len;
2008 h.h1->tp_snaplen = snaplen;
2009 h.h1->tp_mac = macoff;
2010 h.h1->tp_net = netoff;
Daniel Borkmann4b457bd2013-04-16 01:29:11 +00002011 h.h1->tp_sec = ts.tv_sec;
2012 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07002013 hdrlen = sizeof(*h.h1);
2014 break;
2015 case TPACKET_V2:
2016 h.h2->tp_len = skb->len;
2017 h.h2->tp_snaplen = snaplen;
2018 h.h2->tp_mac = macoff;
2019 h.h2->tp_net = netoff;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07002020 h.h2->tp_sec = ts.tv_sec;
2021 h.h2->tp_nsec = ts.tv_nsec;
Jiri Pirkodf8a39d2015-01-13 17:13:44 +01002022 if (skb_vlan_tag_present(skb)) {
2023 h.h2->tp_vlan_tci = skb_vlan_tag_get(skb);
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +09002024 h.h2->tp_vlan_tpid = ntohs(skb->vlan_proto);
2025 status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
Ben Greeara3bcc232011-06-01 06:49:10 +00002026 } else {
2027 h.h2->tp_vlan_tci = 0;
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +09002028 h.h2->tp_vlan_tpid = 0;
Ben Greeara3bcc232011-06-01 06:49:10 +00002029 }
Atzm Watanabee4d26f42013-12-17 22:53:36 +09002030 memset(h.h2->tp_padding, 0, sizeof(h.h2->tp_padding));
Patrick McHardybbd6ef82008-07-14 22:50:15 -07002031 hdrlen = sizeof(*h.h2);
2032 break;
chetan lokef6fb8f102011-08-19 10:18:16 +00002033 case TPACKET_V3:
2034 /* tp_nxt_offset,vlan are already populated above.
2035 * So DONT clear those fields here
2036 */
2037 h.h3->tp_status |= status;
2038 h.h3->tp_len = skb->len;
2039 h.h3->tp_snaplen = snaplen;
2040 h.h3->tp_mac = macoff;
2041 h.h3->tp_net = netoff;
chetan lokef6fb8f102011-08-19 10:18:16 +00002042 h.h3->tp_sec = ts.tv_sec;
2043 h.h3->tp_nsec = ts.tv_nsec;
Atzm Watanabee4d26f42013-12-17 22:53:36 +09002044 memset(h.h3->tp_padding, 0, sizeof(h.h3->tp_padding));
chetan lokef6fb8f102011-08-19 10:18:16 +00002045 hdrlen = sizeof(*h.h3);
2046 break;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07002047 default:
2048 BUG();
2049 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002050
Patrick McHardybbd6ef82008-07-14 22:50:15 -07002051 sll = h.raw + TPACKET_ALIGN(hdrlen);
Stephen Hemmingerb95cce32007-09-26 22:13:38 -07002052 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002053 sll->sll_family = AF_PACKET;
2054 sll->sll_hatype = dev->type;
2055 sll->sll_protocol = skb->protocol;
2056 sll->sll_pkttype = skb->pkt_type;
Peter P Waskiewicz Jr8032b462007-11-10 22:03:25 -08002057 if (unlikely(po->origdev))
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07002058 sll->sll_ifindex = orig_dev->ifindex;
2059 else
2060 sll->sll_ifindex = dev->ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002061
Ralf Baechlee16aa202006-12-07 00:11:33 -08002062 smp_mb();
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01002063
Changli Gaof6dafa92010-12-07 04:26:16 +00002064#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01002065 if (po->tp_version <= TPACKET_V2) {
Changli Gao0af55bb2010-12-01 02:52:20 +00002066 u8 *start, *end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002067
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01002068 end = (u8 *) PAGE_ALIGN((unsigned long) h.raw +
2069 macoff + snaplen);
2070
2071 for (start = h.raw; start < end; start += PAGE_SIZE)
2072 flush_dcache_page(pgv_to_page(start));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002073 }
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01002074 smp_wmb();
Changli Gaof6dafa92010-12-07 04:26:16 +00002075#endif
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01002076
Dan Collinsda413ee2014-12-19 16:49:25 +13002077 if (po->tp_version <= TPACKET_V2) {
chetan lokef6fb8f102011-08-19 10:18:16 +00002078 __packet_set_status(po, h.raw, status);
Dan Collinsda413ee2014-12-19 16:49:25 +13002079 sk->sk_data_ready(sk);
2080 } else {
chetan lokef6fb8f102011-08-19 10:18:16 +00002081 prb_clear_blk_fill_status(&po->rx_ring);
Dan Collinsda413ee2014-12-19 16:49:25 +13002082 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002083
2084drop_n_restore:
2085 if (skb_head != skb->data && skb_shared(skb)) {
2086 skb->data = skb_head;
2087 skb->len = skb_len;
2088 }
2089drop:
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002090 kfree_skb(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002091 return 0;
2092
2093ring_is_full:
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00002094 po->stats.stats1.tp_drops++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002095 spin_unlock(&sk->sk_receive_queue.lock);
2096
David S. Miller676d2362014-04-11 16:15:36 -04002097 sk->sk_data_ready(sk);
Wei Yongjunacb5d752009-02-25 00:36:42 +00002098 kfree_skb(copy_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002099 goto drop_n_restore;
2100}
2101
Johann Baudy69e3c752009-05-18 22:11:22 -07002102static void tpacket_destruct_skb(struct sk_buff *skb)
2103{
2104 struct packet_sock *po = pkt_sk(skb->sk);
Johann Baudy69e3c752009-05-18 22:11:22 -07002105
Johann Baudy69e3c752009-05-18 22:11:22 -07002106 if (likely(po->tx_ring.pg_vec)) {
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01002107 void *ph;
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +00002108 __u32 ts;
2109
Johann Baudy69e3c752009-05-18 22:11:22 -07002110 ph = skb_shinfo(skb)->destructor_arg;
Daniel Borkmannb0138402014-01-15 16:25:36 +01002111 packet_dec_pending(&po->tx_ring);
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +00002112
2113 ts = __packet_set_timestamp(po, ph, skb);
2114 __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
Johann Baudy69e3c752009-05-18 22:11:22 -07002115 }
2116
2117 sock_wfree(skb);
2118}
2119
Willem de Bruijn9c707762014-11-19 13:10:16 -05002120static bool ll_header_truncated(const struct net_device *dev, int len)
2121{
2122 /* net device doesn't like empty head */
2123 if (unlikely(len <= dev->hard_header_len)) {
Willem de Bruijneee2f042015-01-08 11:29:18 -05002124 net_warn_ratelimited("%s: packet size is too short (%d <= %d)\n",
Willem de Bruijn9c707762014-11-19 13:10:16 -05002125 current->comm, len, dev->hard_header_len);
2126 return true;
2127 }
2128
2129 return false;
2130}
2131
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002132static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
2133 void *frame, struct net_device *dev, int size_max,
Herbert Xuae641942011-11-18 02:20:04 +00002134 __be16 proto, unsigned char *addr, int hlen)
Johann Baudy69e3c752009-05-18 22:11:22 -07002135{
Daniel Borkmann184f4892013-04-16 01:57:46 +00002136 union tpacket_uhdr ph;
David S. Miller09effa62013-08-07 17:11:00 -07002137 int to_write, offset, len, tp_len, nr_frags, len_max;
Johann Baudy69e3c752009-05-18 22:11:22 -07002138 struct socket *sock = po->sk.sk_socket;
2139 struct page *page;
2140 void *data;
2141 int err;
2142
2143 ph.raw = frame;
2144
2145 skb->protocol = proto;
2146 skb->dev = dev;
2147 skb->priority = po->sk.sk_priority;
Eric Dumazet2d37a182009-10-01 19:14:46 +00002148 skb->mark = po->sk.sk_mark;
Willem de Bruijn2e313962013-04-23 00:39:28 +00002149 sock_tx_timestamp(&po->sk, &skb_shinfo(skb)->tx_flags);
Johann Baudy69e3c752009-05-18 22:11:22 -07002150 skb_shinfo(skb)->destructor_arg = ph.raw;
2151
2152 switch (po->tp_version) {
2153 case TPACKET_V2:
2154 tp_len = ph.h2->tp_len;
2155 break;
2156 default:
2157 tp_len = ph.h1->tp_len;
2158 break;
2159 }
David S. Miller09effa62013-08-07 17:11:00 -07002160 if (unlikely(tp_len > size_max)) {
2161 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
2162 return -EMSGSIZE;
2163 }
Johann Baudy69e3c752009-05-18 22:11:22 -07002164
Herbert Xuae641942011-11-18 02:20:04 +00002165 skb_reserve(skb, hlen);
Johann Baudy69e3c752009-05-18 22:11:22 -07002166 skb_reset_network_header(skb);
Jason Wangc1aad272013-03-25 20:19:57 +00002167
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01002168 if (!packet_use_direct_xmit(po))
2169 skb_probe_transport_header(skb, 0);
2170 if (unlikely(po->tp_tx_has_off)) {
Paul Chavent5920cd3a2012-11-06 23:10:47 +00002171 int off_min, off_max, off;
2172 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2173 off_max = po->tx_ring.frame_size - tp_len;
2174 if (sock->type == SOCK_DGRAM) {
2175 switch (po->tp_version) {
2176 case TPACKET_V2:
2177 off = ph.h2->tp_net;
2178 break;
2179 default:
2180 off = ph.h1->tp_net;
2181 break;
2182 }
2183 } else {
2184 switch (po->tp_version) {
2185 case TPACKET_V2:
2186 off = ph.h2->tp_mac;
2187 break;
2188 default:
2189 off = ph.h1->tp_mac;
2190 break;
2191 }
2192 }
2193 if (unlikely((off < off_min) || (off_max < off)))
2194 return -EINVAL;
2195 data = ph.raw + off;
2196 } else {
2197 data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll);
2198 }
Johann Baudy69e3c752009-05-18 22:11:22 -07002199 to_write = tp_len;
2200
2201 if (sock->type == SOCK_DGRAM) {
2202 err = dev_hard_header(skb, dev, ntohs(proto), addr,
2203 NULL, tp_len);
2204 if (unlikely(err < 0))
2205 return -EINVAL;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002206 } else if (dev->hard_header_len) {
Willem de Bruijn9c707762014-11-19 13:10:16 -05002207 if (ll_header_truncated(dev, tp_len))
Johann Baudy69e3c752009-05-18 22:11:22 -07002208 return -EINVAL;
Johann Baudy69e3c752009-05-18 22:11:22 -07002209
2210 skb_push(skb, dev->hard_header_len);
2211 err = skb_store_bits(skb, 0, data,
2212 dev->hard_header_len);
2213 if (unlikely(err))
2214 return err;
2215
2216 data += dev->hard_header_len;
2217 to_write -= dev->hard_header_len;
2218 }
2219
Johann Baudy69e3c752009-05-18 22:11:22 -07002220 offset = offset_in_page(data);
2221 len_max = PAGE_SIZE - offset;
2222 len = ((to_write > len_max) ? len_max : to_write);
2223
2224 skb->data_len = to_write;
2225 skb->len += to_write;
2226 skb->truesize += to_write;
2227 atomic_add(to_write, &po->sk.sk_wmem_alloc);
2228
2229 while (likely(to_write)) {
2230 nr_frags = skb_shinfo(skb)->nr_frags;
2231
2232 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002233 pr_err("Packet exceed the number of skb frags(%lu)\n",
2234 MAX_SKB_FRAGS);
Johann Baudy69e3c752009-05-18 22:11:22 -07002235 return -EFAULT;
2236 }
2237
Changli Gao0af55bb2010-12-01 02:52:20 +00002238 page = pgv_to_page(data);
2239 data += len;
Johann Baudy69e3c752009-05-18 22:11:22 -07002240 flush_dcache_page(page);
2241 get_page(page);
Changli Gao0af55bb2010-12-01 02:52:20 +00002242 skb_fill_page_desc(skb, nr_frags, page, offset, len);
Johann Baudy69e3c752009-05-18 22:11:22 -07002243 to_write -= len;
2244 offset = 0;
2245 len_max = PAGE_SIZE;
2246 len = ((to_write > len_max) ? len_max : to_write);
2247 }
2248
2249 return tp_len;
2250}
2251
2252static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2253{
Johann Baudy69e3c752009-05-18 22:11:22 -07002254 struct sk_buff *skb;
2255 struct net_device *dev;
2256 __be16 proto;
David S. Miller09effa62013-08-07 17:11:00 -07002257 int err, reserve = 0;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002258 void *ph;
Steffen Hurrle342dfc32014-01-17 22:53:15 +01002259 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
Daniel Borkmann87a2fd22014-01-15 16:25:35 +01002260 bool need_wait = !(msg->msg_flags & MSG_DONTWAIT);
Johann Baudy69e3c752009-05-18 22:11:22 -07002261 int tp_len, size_max;
2262 unsigned char *addr;
2263 int len_sum = 0;
danborkmann@iogearbox.net9e670302012-08-20 03:34:03 +00002264 int status = TP_STATUS_AVAILABLE;
Herbert Xuae641942011-11-18 02:20:04 +00002265 int hlen, tlen;
Johann Baudy69e3c752009-05-18 22:11:22 -07002266
Johann Baudy69e3c752009-05-18 22:11:22 -07002267 mutex_lock(&po->pg_vec_lock);
2268
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01002269 if (likely(saddr == NULL)) {
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002270 dev = packet_cached_dev_get(po);
Johann Baudy69e3c752009-05-18 22:11:22 -07002271 proto = po->num;
2272 addr = NULL;
2273 } else {
2274 err = -EINVAL;
2275 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2276 goto out;
2277 if (msg->msg_namelen < (saddr->sll_halen
2278 + offsetof(struct sockaddr_ll,
2279 sll_addr)))
2280 goto out;
Johann Baudy69e3c752009-05-18 22:11:22 -07002281 proto = saddr->sll_protocol;
2282 addr = saddr->sll_addr;
Ben Greear827d9782011-06-01 07:18:53 +00002283 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
Johann Baudy69e3c752009-05-18 22:11:22 -07002284 }
2285
Johann Baudy69e3c752009-05-18 22:11:22 -07002286 err = -ENXIO;
2287 if (unlikely(dev == NULL))
2288 goto out;
Johann Baudy69e3c752009-05-18 22:11:22 -07002289 err = -ENETDOWN;
2290 if (unlikely(!(dev->flags & IFF_UP)))
2291 goto out_put;
2292
Daniel Borkmann52f14542014-02-28 02:22:06 +01002293 reserve = dev->hard_header_len + VLAN_HLEN;
Johann Baudy69e3c752009-05-18 22:11:22 -07002294 size_max = po->tx_ring.frame_size
Gabor Gombasb5dd8842009-10-29 03:19:11 -07002295 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
Johann Baudy69e3c752009-05-18 22:11:22 -07002296
David S. Miller09effa62013-08-07 17:11:00 -07002297 if (size_max > dev->mtu + reserve)
2298 size_max = dev->mtu + reserve;
2299
Johann Baudy69e3c752009-05-18 22:11:22 -07002300 do {
2301 ph = packet_current_frame(po, &po->tx_ring,
Daniel Borkmann87a2fd22014-01-15 16:25:35 +01002302 TP_STATUS_SEND_REQUEST);
Johann Baudy69e3c752009-05-18 22:11:22 -07002303 if (unlikely(ph == NULL)) {
Daniel Borkmann87a2fd22014-01-15 16:25:35 +01002304 if (need_wait && need_resched())
2305 schedule();
Johann Baudy69e3c752009-05-18 22:11:22 -07002306 continue;
2307 }
2308
2309 status = TP_STATUS_SEND_REQUEST;
Herbert Xuae641942011-11-18 02:20:04 +00002310 hlen = LL_RESERVED_SPACE(dev);
2311 tlen = dev->needed_tailroom;
Johann Baudy69e3c752009-05-18 22:11:22 -07002312 skb = sock_alloc_send_skb(&po->sk,
Herbert Xuae641942011-11-18 02:20:04 +00002313 hlen + tlen + sizeof(struct sockaddr_ll),
Johann Baudy69e3c752009-05-18 22:11:22 -07002314 0, &err);
2315
2316 if (unlikely(skb == NULL))
2317 goto out_status;
2318
2319 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto,
Daniel Borkmann52f14542014-02-28 02:22:06 +01002320 addr, hlen);
2321 if (tp_len > dev->mtu + dev->hard_header_len) {
2322 struct ethhdr *ehdr;
2323 /* Earlier code assumed this would be a VLAN pkt,
2324 * double-check this now that we have the actual
2325 * packet in hand.
2326 */
Johann Baudy69e3c752009-05-18 22:11:22 -07002327
Daniel Borkmann52f14542014-02-28 02:22:06 +01002328 skb_reset_mac_header(skb);
2329 ehdr = eth_hdr(skb);
2330 if (ehdr->h_proto != htons(ETH_P_8021Q))
2331 tp_len = -EMSGSIZE;
2332 }
Johann Baudy69e3c752009-05-18 22:11:22 -07002333 if (unlikely(tp_len < 0)) {
2334 if (po->tp_loss) {
2335 __packet_set_status(po, ph,
2336 TP_STATUS_AVAILABLE);
2337 packet_increment_head(&po->tx_ring);
2338 kfree_skb(skb);
2339 continue;
2340 } else {
2341 status = TP_STATUS_WRONG_FORMAT;
2342 err = tp_len;
2343 goto out_status;
2344 }
2345 }
2346
Daniel Borkmann0fd5d572014-02-16 15:55:22 +01002347 packet_pick_tx_queue(dev, skb);
2348
Johann Baudy69e3c752009-05-18 22:11:22 -07002349 skb->destructor = tpacket_destruct_skb;
2350 __packet_set_status(po, ph, TP_STATUS_SENDING);
Daniel Borkmannb0138402014-01-15 16:25:36 +01002351 packet_inc_pending(&po->tx_ring);
Johann Baudy69e3c752009-05-18 22:11:22 -07002352
2353 status = TP_STATUS_SEND_REQUEST;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01002354 err = po->xmit(skb);
Jarek Poplawskieb70df12010-01-10 22:04:19 +00002355 if (unlikely(err > 0)) {
2356 err = net_xmit_errno(err);
2357 if (err && __packet_get_status(po, ph) ==
2358 TP_STATUS_AVAILABLE) {
2359 /* skb was destructed already */
2360 skb = NULL;
2361 goto out_status;
2362 }
2363 /*
2364 * skb was dropped but not destructed yet;
2365 * let's treat it like congestion or err < 0
2366 */
2367 err = 0;
2368 }
Johann Baudy69e3c752009-05-18 22:11:22 -07002369 packet_increment_head(&po->tx_ring);
2370 len_sum += tp_len;
Daniel Borkmannb0138402014-01-15 16:25:36 +01002371 } while (likely((ph != NULL) ||
2372 /* Note: packet_read_pending() might be slow if we have
2373 * to call it as it's per_cpu variable, but in fast-path
2374 * we already short-circuit the loop with the first
2375 * condition, and luckily don't have to go that path
2376 * anyway.
2377 */
2378 (need_wait && packet_read_pending(&po->tx_ring))));
Johann Baudy69e3c752009-05-18 22:11:22 -07002379
2380 err = len_sum;
2381 goto out_put;
2382
Johann Baudy69e3c752009-05-18 22:11:22 -07002383out_status:
2384 __packet_set_status(po, ph, status);
2385 kfree_skb(skb);
2386out_put:
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002387 dev_put(dev);
Johann Baudy69e3c752009-05-18 22:11:22 -07002388out:
2389 mutex_unlock(&po->pg_vec_lock);
2390 return err;
2391}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002392
Olof Johanssoneea49cc92011-11-02 11:00:49 +00002393static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2394 size_t reserve, size_t len,
2395 size_t linear, int noblock,
2396 int *err)
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002397{
2398 struct sk_buff *skb;
2399
2400 /* Under a page? Don't bother with paged skb. */
2401 if (prepad + len < PAGE_SIZE || !linear)
2402 linear = len;
2403
2404 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
Eric Dumazet28d64272013-08-08 14:38:47 -07002405 err, 0);
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002406 if (!skb)
2407 return NULL;
2408
2409 skb_reserve(skb, reserve);
2410 skb_put(skb, linear);
2411 skb->data_len = len - linear;
2412 skb->len += len - linear;
2413
2414 return skb;
2415}
2416
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01002417static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002418{
2419 struct sock *sk = sock->sk;
Steffen Hurrle342dfc32014-01-17 22:53:15 +01002420 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002421 struct sk_buff *skb;
2422 struct net_device *dev;
Al Viro0e11c912006-11-08 00:26:29 -08002423 __be16 proto;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002424 unsigned char *addr;
Ben Greear827d9782011-06-01 07:18:53 +00002425 int err, reserve = 0;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002426 struct virtio_net_hdr vnet_hdr = { 0 };
2427 int offset = 0;
2428 int vnet_hdr_len;
2429 struct packet_sock *po = pkt_sk(sk);
2430 unsigned short gso_type = 0;
Herbert Xuae641942011-11-18 02:20:04 +00002431 int hlen, tlen;
Ben Greear3bdc0eb2012-02-11 15:39:30 +00002432 int extra_len = 0;
Al Viro8feb2fb2014-11-06 01:10:59 -05002433 ssize_t n;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002434
2435 /*
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002436 * Get and verify the address.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002437 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002438
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01002439 if (likely(saddr == NULL)) {
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002440 dev = packet_cached_dev_get(po);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002441 proto = po->num;
2442 addr = NULL;
2443 } else {
2444 err = -EINVAL;
2445 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2446 goto out;
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07002447 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2448 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002449 proto = saddr->sll_protocol;
2450 addr = saddr->sll_addr;
Ben Greear827d9782011-06-01 07:18:53 +00002451 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002452 }
2453
Linus Torvalds1da177e2005-04-16 15:20:36 -07002454 err = -ENXIO;
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002455 if (unlikely(dev == NULL))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002456 goto out_unlock;
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002457 err = -ENETDOWN;
2458 if (unlikely(!(dev->flags & IFF_UP)))
2459 goto out_unlock;
2460
Linus Torvalds1da177e2005-04-16 15:20:36 -07002461 if (sock->type == SOCK_RAW)
2462 reserve = dev->hard_header_len;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002463 if (po->has_vnet_hdr) {
2464 vnet_hdr_len = sizeof(vnet_hdr);
2465
2466 err = -EINVAL;
2467 if (len < vnet_hdr_len)
2468 goto out_unlock;
2469
2470 len -= vnet_hdr_len;
2471
Al Viro8feb2fb2014-11-06 01:10:59 -05002472 err = -EFAULT;
Al Viroc0371da2014-11-24 10:42:55 -05002473 n = copy_from_iter(&vnet_hdr, vnet_hdr_len, &msg->msg_iter);
Al Viro8feb2fb2014-11-06 01:10:59 -05002474 if (n != vnet_hdr_len)
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002475 goto out_unlock;
2476
2477 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
Michael S. Tsirkindc9e5152014-11-23 18:16:19 +02002478 (__virtio16_to_cpu(false, vnet_hdr.csum_start) +
2479 __virtio16_to_cpu(false, vnet_hdr.csum_offset) + 2 >
2480 __virtio16_to_cpu(false, vnet_hdr.hdr_len)))
2481 vnet_hdr.hdr_len = __cpu_to_virtio16(false,
2482 __virtio16_to_cpu(false, vnet_hdr.csum_start) +
2483 __virtio16_to_cpu(false, vnet_hdr.csum_offset) + 2);
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002484
2485 err = -EINVAL;
Michael S. Tsirkindc9e5152014-11-23 18:16:19 +02002486 if (__virtio16_to_cpu(false, vnet_hdr.hdr_len) > len)
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002487 goto out_unlock;
2488
2489 if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
2490 switch (vnet_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2491 case VIRTIO_NET_HDR_GSO_TCPV4:
2492 gso_type = SKB_GSO_TCPV4;
2493 break;
2494 case VIRTIO_NET_HDR_GSO_TCPV6:
2495 gso_type = SKB_GSO_TCPV6;
2496 break;
2497 case VIRTIO_NET_HDR_GSO_UDP:
2498 gso_type = SKB_GSO_UDP;
2499 break;
2500 default:
2501 goto out_unlock;
2502 }
2503
2504 if (vnet_hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
2505 gso_type |= SKB_GSO_TCP_ECN;
2506
2507 if (vnet_hdr.gso_size == 0)
2508 goto out_unlock;
2509
2510 }
2511 }
2512
Ben Greear3bdc0eb2012-02-11 15:39:30 +00002513 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
2514 if (!netif_supports_nofcs(dev)) {
2515 err = -EPROTONOSUPPORT;
2516 goto out_unlock;
2517 }
2518 extra_len = 4; /* We're doing our own CRC */
2519 }
2520
Linus Torvalds1da177e2005-04-16 15:20:36 -07002521 err = -EMSGSIZE;
Ben Greear3bdc0eb2012-02-11 15:39:30 +00002522 if (!gso_type && (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002523 goto out_unlock;
2524
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002525 err = -ENOBUFS;
Herbert Xuae641942011-11-18 02:20:04 +00002526 hlen = LL_RESERVED_SPACE(dev);
2527 tlen = dev->needed_tailroom;
Michael S. Tsirkindc9e5152014-11-23 18:16:19 +02002528 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len,
2529 __virtio16_to_cpu(false, vnet_hdr.hdr_len),
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002530 msg->msg_flags & MSG_DONTWAIT, &err);
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002531 if (skb == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002532 goto out_unlock;
2533
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002534 skb_set_network_header(skb, reserve);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002535
Stephen Hemminger0c4e8582007-10-09 01:36:32 -07002536 err = -EINVAL;
Willem de Bruijn9c707762014-11-19 13:10:16 -05002537 if (sock->type == SOCK_DGRAM) {
2538 offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len);
Christoph Jaeger46d2cfb2015-01-11 13:01:16 -05002539 if (unlikely(offset < 0))
Willem de Bruijn9c707762014-11-19 13:10:16 -05002540 goto out_free;
2541 } else {
2542 if (ll_header_truncated(dev, len))
2543 goto out_free;
2544 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002545
2546 /* Returns -EFAULT on error */
Al Viroc0371da2014-11-24 10:42:55 -05002547 err = skb_copy_datagram_from_iter(skb, offset, &msg->msg_iter, len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002548 if (err)
2549 goto out_free;
Daniel Borkmannbf84a012013-04-14 08:08:13 +00002550
2551 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002552
Ben Greear3bdc0eb2012-02-11 15:39:30 +00002553 if (!gso_type && (len > dev->mtu + reserve + extra_len)) {
David S. Miller09effa62013-08-07 17:11:00 -07002554 /* Earlier code assumed this would be a VLAN pkt,
2555 * double-check this now that we have the actual
2556 * packet in hand.
2557 */
2558 struct ethhdr *ehdr;
2559 skb_reset_mac_header(skb);
2560 ehdr = eth_hdr(skb);
2561 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
2562 err = -EMSGSIZE;
2563 goto out_free;
2564 }
Ben Greear57f89bf2011-02-11 09:35:18 +00002565 }
2566
David S. Miller09effa62013-08-07 17:11:00 -07002567 skb->protocol = proto;
2568 skb->dev = dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002569 skb->priority = sk->sk_priority;
Eric Dumazet2d37a182009-10-01 19:14:46 +00002570 skb->mark = sk->sk_mark;
Daniel Borkmann0fd5d572014-02-16 15:55:22 +01002571
2572 packet_pick_tx_queue(dev, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002573
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002574 if (po->has_vnet_hdr) {
2575 if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
Michael S. Tsirkindc9e5152014-11-23 18:16:19 +02002576 u16 s = __virtio16_to_cpu(false, vnet_hdr.csum_start);
2577 u16 o = __virtio16_to_cpu(false, vnet_hdr.csum_offset);
2578 if (!skb_partial_csum_set(skb, s, o)) {
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002579 err = -EINVAL;
2580 goto out_free;
2581 }
2582 }
2583
Michael S. Tsirkindc9e5152014-11-23 18:16:19 +02002584 skb_shinfo(skb)->gso_size =
2585 __virtio16_to_cpu(false, vnet_hdr.gso_size);
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002586 skb_shinfo(skb)->gso_type = gso_type;
2587
2588 /* Header must be checked, and gso_segs computed. */
2589 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2590 skb_shinfo(skb)->gso_segs = 0;
2591
2592 len += vnet_hdr_len;
2593 }
2594
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01002595 if (!packet_use_direct_xmit(po))
2596 skb_probe_transport_header(skb, reserve);
Ben Greear3bdc0eb2012-02-11 15:39:30 +00002597 if (unlikely(extra_len == 4))
2598 skb->no_fcs = 1;
2599
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01002600 err = po->xmit(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002601 if (err > 0 && (err = net_xmit_errno(err)) != 0)
2602 goto out_unlock;
2603
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002604 dev_put(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002605
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002606 return len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002607
2608out_free:
2609 kfree_skb(skb);
2610out_unlock:
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002611 if (dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002612 dev_put(dev);
2613out:
2614 return err;
2615}
2616
Ying Xue1b784142015-03-02 15:37:48 +08002617static int packet_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
Johann Baudy69e3c752009-05-18 22:11:22 -07002618{
Johann Baudy69e3c752009-05-18 22:11:22 -07002619 struct sock *sk = sock->sk;
2620 struct packet_sock *po = pkt_sk(sk);
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01002621
Johann Baudy69e3c752009-05-18 22:11:22 -07002622 if (po->tx_ring.pg_vec)
2623 return tpacket_snd(po, msg);
2624 else
Johann Baudy69e3c752009-05-18 22:11:22 -07002625 return packet_snd(sock, msg, len);
2626}
2627
Linus Torvalds1da177e2005-04-16 15:20:36 -07002628/*
2629 * Close a PACKET socket. This is fairly simple. We immediately go
2630 * to 'closed' state and remove our protocol entry in the device list.
2631 */
2632
2633static int packet_release(struct socket *sock)
2634{
2635 struct sock *sk = sock->sk;
2636 struct packet_sock *po;
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08002637 struct net *net;
chetan lokef6fb8f102011-08-19 10:18:16 +00002638 union tpacket_req_u req_u;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002639
2640 if (!sk)
2641 return 0;
2642
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09002643 net = sock_net(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002644 po = pkt_sk(sk);
2645
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00002646 mutex_lock(&net->packet.sklist_lock);
stephen hemminger808f5112010-02-22 07:57:18 +00002647 sk_del_node_init_rcu(sk);
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00002648 mutex_unlock(&net->packet.sklist_lock);
2649
2650 preempt_disable();
Eric Dumazet920de802008-11-24 00:09:29 -08002651 sock_prot_inuse_add(net, sk->sk_prot, -1);
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00002652 preempt_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002653
stephen hemminger808f5112010-02-22 07:57:18 +00002654 spin_lock(&po->bind_lock);
David S. Millerce06b032011-07-04 01:44:29 -07002655 unregister_prot_hook(sk, false);
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01002656 packet_cached_dev_reset(po);
2657
Ben Greear160ff182011-06-01 07:18:52 +00002658 if (po->prot_hook.dev) {
2659 dev_put(po->prot_hook.dev);
2660 po->prot_hook.dev = NULL;
2661 }
stephen hemminger808f5112010-02-22 07:57:18 +00002662 spin_unlock(&po->bind_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002663
Linus Torvalds1da177e2005-04-16 15:20:36 -07002664 packet_flush_mclist(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002665
Phil Sutter9665d5d2013-02-01 07:21:41 +00002666 if (po->rx_ring.pg_vec) {
2667 memset(&req_u, 0, sizeof(req_u));
chetan lokef6fb8f102011-08-19 10:18:16 +00002668 packet_set_ring(sk, &req_u, 1, 0);
Phil Sutter9665d5d2013-02-01 07:21:41 +00002669 }
Johann Baudy69e3c752009-05-18 22:11:22 -07002670
Phil Sutter9665d5d2013-02-01 07:21:41 +00002671 if (po->tx_ring.pg_vec) {
2672 memset(&req_u, 0, sizeof(req_u));
chetan lokef6fb8f102011-08-19 10:18:16 +00002673 packet_set_ring(sk, &req_u, 1, 1);
Phil Sutter9665d5d2013-02-01 07:21:41 +00002674 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002675
David S. Millerdc99f602011-07-05 01:45:05 -07002676 fanout_release(sk);
2677
stephen hemminger808f5112010-02-22 07:57:18 +00002678 synchronize_net();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002679 /*
2680 * Now the socket is dead. No more input will appear.
2681 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002682 sock_orphan(sk);
2683 sock->sk = NULL;
2684
2685 /* Purge queues */
2686
2687 skb_queue_purge(&sk->sk_receive_queue);
Daniel Borkmannb0138402014-01-15 16:25:36 +01002688 packet_free_pending(po);
Pavel Emelyanov17ab56a2007-11-10 21:38:48 -08002689 sk_refcnt_debug_release(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002690
2691 sock_put(sk);
2692 return 0;
2693}
2694
2695/*
2696 * Attach a packet hook.
2697 */
2698
Daniel Borkmann902fefb2014-01-15 16:25:34 +01002699static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 proto)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002700{
2701 struct packet_sock *po = pkt_sk(sk);
Daniel Borkmann902fefb2014-01-15 16:25:34 +01002702 const struct net_device *dev_curr;
2703 __be16 proto_curr;
2704 bool need_rehook;
David S. Millerdc99f602011-07-05 01:45:05 -07002705
Wei Yongjunaef950b2011-12-27 22:32:41 -05002706 if (po->fanout) {
2707 if (dev)
2708 dev_put(dev);
2709
David S. Millerdc99f602011-07-05 01:45:05 -07002710 return -EINVAL;
Wei Yongjunaef950b2011-12-27 22:32:41 -05002711 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002712
2713 lock_sock(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002714 spin_lock(&po->bind_lock);
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01002715
Daniel Borkmann902fefb2014-01-15 16:25:34 +01002716 proto_curr = po->prot_hook.type;
2717 dev_curr = po->prot_hook.dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002718
Daniel Borkmann902fefb2014-01-15 16:25:34 +01002719 need_rehook = proto_curr != proto || dev_curr != dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002720
Daniel Borkmann902fefb2014-01-15 16:25:34 +01002721 if (need_rehook) {
2722 unregister_prot_hook(sk, true);
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01002723
Daniel Borkmann902fefb2014-01-15 16:25:34 +01002724 po->num = proto;
2725 po->prot_hook.type = proto;
2726
2727 if (po->prot_hook.dev)
2728 dev_put(po->prot_hook.dev);
2729
2730 po->prot_hook.dev = dev;
2731
2732 po->ifindex = dev ? dev->ifindex : 0;
2733 packet_cached_dev_assign(po, dev);
2734 }
2735
2736 if (proto == 0 || !need_rehook)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002737 goto out_unlock;
2738
Urs Thuermannbe85d4a2007-11-12 21:05:20 -08002739 if (!dev || (dev->flags & IFF_UP)) {
David S. Millerce06b032011-07-04 01:44:29 -07002740 register_prot_hook(sk);
Urs Thuermannbe85d4a2007-11-12 21:05:20 -08002741 } else {
2742 sk->sk_err = ENETDOWN;
2743 if (!sock_flag(sk, SOCK_DEAD))
2744 sk->sk_error_report(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002745 }
2746
2747out_unlock:
2748 spin_unlock(&po->bind_lock);
2749 release_sock(sk);
2750 return 0;
2751}
2752
2753/*
2754 * Bind a packet socket to a device
2755 */
2756
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002757static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
2758 int addr_len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002759{
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002760 struct sock *sk = sock->sk;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002761 char name[15];
2762 struct net_device *dev;
2763 int err = -ENODEV;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002764
Linus Torvalds1da177e2005-04-16 15:20:36 -07002765 /*
2766 * Check legality
2767 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002768
Kris Katterjohn8ae55f02006-01-23 16:28:02 -08002769 if (addr_len != sizeof(struct sockaddr))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002770 return -EINVAL;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002771 strlcpy(name, uaddr->sa_data, sizeof(name));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002772
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09002773 dev = dev_get_by_name(sock_net(sk), name);
Ben Greear160ff182011-06-01 07:18:52 +00002774 if (dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002775 err = packet_do_bind(sk, dev, pkt_sk(sk)->num);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002776 return err;
2777}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002778
2779static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
2780{
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002781 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
2782 struct sock *sk = sock->sk;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002783 struct net_device *dev = NULL;
2784 int err;
2785
2786
2787 /*
2788 * Check legality
2789 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002790
Linus Torvalds1da177e2005-04-16 15:20:36 -07002791 if (addr_len < sizeof(struct sockaddr_ll))
2792 return -EINVAL;
2793 if (sll->sll_family != AF_PACKET)
2794 return -EINVAL;
2795
2796 if (sll->sll_ifindex) {
2797 err = -ENODEV;
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09002798 dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002799 if (dev == NULL)
2800 goto out;
2801 }
2802 err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002803
2804out:
2805 return err;
2806}
2807
2808static struct proto packet_proto = {
2809 .name = "PACKET",
2810 .owner = THIS_MODULE,
2811 .obj_size = sizeof(struct packet_sock),
2812};
2813
2814/*
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002815 * Create a packet of type SOCK_PACKET.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002816 */
2817
Eric Paris3f378b62009-11-05 22:18:14 -08002818static int packet_create(struct net *net, struct socket *sock, int protocol,
2819 int kern)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002820{
2821 struct sock *sk;
2822 struct packet_sock *po;
Al Viro0e11c912006-11-08 00:26:29 -08002823 __be16 proto = (__force __be16)protocol; /* weird, but documented */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002824 int err;
2825
Eric W. Biedermandf008c92012-11-16 03:03:07 +00002826 if (!ns_capable(net->user_ns, CAP_NET_RAW))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002827 return -EPERM;
David S. Millerbe020972007-05-29 13:16:31 -07002828 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
2829 sock->type != SOCK_PACKET)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002830 return -ESOCKTNOSUPPORT;
2831
2832 sock->state = SS_UNCONNECTED;
2833
2834 err = -ENOBUFS;
Pavel Emelyanov6257ff22007-11-01 00:39:31 -07002835 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002836 if (sk == NULL)
2837 goto out;
2838
2839 sock->ops = &packet_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002840 if (sock->type == SOCK_PACKET)
2841 sock->ops = &packet_ops_spkt;
David S. Millerbe020972007-05-29 13:16:31 -07002842
Linus Torvalds1da177e2005-04-16 15:20:36 -07002843 sock_init_data(sock, sk);
2844
2845 po = pkt_sk(sk);
2846 sk->sk_family = PF_PACKET;
Al Viro0e11c912006-11-08 00:26:29 -08002847 po->num = proto;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01002848 po->xmit = dev_queue_xmit;
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01002849
Daniel Borkmannb0138402014-01-15 16:25:36 +01002850 err = packet_alloc_pending(po);
2851 if (err)
2852 goto out2;
2853
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01002854 packet_cached_dev_reset(po);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002855
2856 sk->sk_destruct = packet_sock_destruct;
Pavel Emelyanov17ab56a2007-11-10 21:38:48 -08002857 sk_refcnt_debug_inc(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002858
2859 /*
2860 * Attach a protocol block
2861 */
2862
2863 spin_lock_init(&po->bind_lock);
Herbert Xu905db442009-01-30 14:12:06 -08002864 mutex_init(&po->pg_vec_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002865 po->prot_hook.func = packet_rcv;
David S. Millerbe020972007-05-29 13:16:31 -07002866
Linus Torvalds1da177e2005-04-16 15:20:36 -07002867 if (sock->type == SOCK_PACKET)
2868 po->prot_hook.func = packet_rcv_spkt;
David S. Millerbe020972007-05-29 13:16:31 -07002869
Linus Torvalds1da177e2005-04-16 15:20:36 -07002870 po->prot_hook.af_packet_priv = sk;
2871
Al Viro0e11c912006-11-08 00:26:29 -08002872 if (proto) {
2873 po->prot_hook.type = proto;
David S. Millerce06b032011-07-04 01:44:29 -07002874 register_prot_hook(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002875 }
2876
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00002877 mutex_lock(&net->packet.sklist_lock);
stephen hemminger808f5112010-02-22 07:57:18 +00002878 sk_add_node_rcu(sk, &net->packet.sklist);
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00002879 mutex_unlock(&net->packet.sklist_lock);
2880
2881 preempt_disable();
Eric Dumazet36804532008-11-19 14:25:35 -08002882 sock_prot_inuse_add(net, &packet_proto, 1);
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00002883 preempt_enable();
stephen hemminger808f5112010-02-22 07:57:18 +00002884
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002885 return 0;
Daniel Borkmannb0138402014-01-15 16:25:36 +01002886out2:
2887 sk_free(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002888out:
2889 return err;
2890}
2891
2892/*
2893 * Pull a packet from our receive queue and hand it to the user.
2894 * If necessary we block.
2895 */
2896
Ying Xue1b784142015-03-02 15:37:48 +08002897static int packet_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
2898 int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002899{
2900 struct sock *sk = sock->sk;
2901 struct sk_buff *skb;
2902 int copied, err;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002903 int vnet_hdr_len = 0;
Eyal Birger2472d762015-03-01 14:58:28 +02002904 unsigned int origlen = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002905
2906 err = -EINVAL;
Richard Cochraned85b562010-04-07 22:41:28 +00002907 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002908 goto out;
2909
2910#if 0
2911 /* What error should we return now? EUNATTACH? */
2912 if (pkt_sk(sk)->ifindex < 0)
2913 return -ENODEV;
2914#endif
2915
Richard Cochraned85b562010-04-07 22:41:28 +00002916 if (flags & MSG_ERRQUEUE) {
Richard Cochrancb820f82013-07-19 19:40:09 +02002917 err = sock_recv_errqueue(sk, msg, len,
2918 SOL_PACKET, PACKET_TX_TIMESTAMP);
Richard Cochraned85b562010-04-07 22:41:28 +00002919 goto out;
2920 }
2921
Linus Torvalds1da177e2005-04-16 15:20:36 -07002922 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -07002923 * Call the generic datagram receiver. This handles all sorts
2924 * of horrible races and re-entrancy so we can forget about it
2925 * in the protocol layers.
2926 *
2927 * Now it will return ENETDOWN, if device have just gone down,
2928 * but then it will block.
2929 */
2930
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002931 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002932
2933 /*
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002934 * An error occurred so return it. Because skb_recv_datagram()
Linus Torvalds1da177e2005-04-16 15:20:36 -07002935 * handles the blocking we don't see and worry about blocking
2936 * retries.
2937 */
2938
Kris Katterjohn8ae55f02006-01-23 16:28:02 -08002939 if (skb == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002940 goto out;
2941
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002942 if (pkt_sk(sk)->has_vnet_hdr) {
2943 struct virtio_net_hdr vnet_hdr = { 0 };
2944
2945 err = -EINVAL;
2946 vnet_hdr_len = sizeof(vnet_hdr);
Mariusz Kozlowski1f18b712010-11-08 11:58:45 +00002947 if (len < vnet_hdr_len)
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002948 goto out_free;
2949
Mariusz Kozlowski1f18b712010-11-08 11:58:45 +00002950 len -= vnet_hdr_len;
2951
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002952 if (skb_is_gso(skb)) {
2953 struct skb_shared_info *sinfo = skb_shinfo(skb);
2954
2955 /* This is a hint as to how much should be linear. */
Michael S. Tsirkindc9e5152014-11-23 18:16:19 +02002956 vnet_hdr.hdr_len =
2957 __cpu_to_virtio16(false, skb_headlen(skb));
2958 vnet_hdr.gso_size =
2959 __cpu_to_virtio16(false, sinfo->gso_size);
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002960 if (sinfo->gso_type & SKB_GSO_TCPV4)
2961 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
2962 else if (sinfo->gso_type & SKB_GSO_TCPV6)
2963 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
2964 else if (sinfo->gso_type & SKB_GSO_UDP)
2965 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
2966 else if (sinfo->gso_type & SKB_GSO_FCOE)
2967 goto out_free;
2968 else
2969 BUG();
2970 if (sinfo->gso_type & SKB_GSO_TCP_ECN)
2971 vnet_hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
2972 } else
2973 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
2974
2975 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2976 vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
Michael S. Tsirkindc9e5152014-11-23 18:16:19 +02002977 vnet_hdr.csum_start = __cpu_to_virtio16(false,
2978 skb_checksum_start_offset(skb));
2979 vnet_hdr.csum_offset = __cpu_to_virtio16(false,
2980 skb->csum_offset);
Jason Wang10a8d942011-06-10 00:56:17 +00002981 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
2982 vnet_hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002983 } /* else everything is zero */
2984
Al Viro7eab8d92014-04-06 21:51:23 -04002985 err = memcpy_to_msg(msg, (void *)&vnet_hdr, vnet_hdr_len);
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002986 if (err < 0)
2987 goto out_free;
2988 }
2989
Hannes Frederic Sowaf3d33422013-11-21 03:14:22 +01002990 /* You lose any data beyond the buffer you gave. If it worries
2991 * a user program they can ask the device for its MTU
2992 * anyway.
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07002993 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002994 copied = skb->len;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002995 if (copied > len) {
2996 copied = len;
2997 msg->msg_flags |= MSG_TRUNC;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002998 }
2999
David S. Miller51f3d022014-11-05 16:46:40 -05003000 err = skb_copy_datagram_msg(skb, 0, msg, copied);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003001 if (err)
3002 goto out_free;
3003
Eyal Birger2472d762015-03-01 14:58:28 +02003004 if (sock->type != SOCK_PACKET) {
3005 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
3006
3007 /* Original length was stored in sockaddr_ll fields */
3008 origlen = PACKET_SKB_CB(skb)->sa.origlen;
3009 sll->sll_family = AF_PACKET;
3010 sll->sll_protocol = skb->protocol;
3011 }
3012
Neil Horman3b885782009-10-12 13:26:31 -07003013 sock_recv_ts_and_drops(msg, sk, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003014
Hannes Frederic Sowaf3d33422013-11-21 03:14:22 +01003015 if (msg->msg_name) {
3016 /* If the address length field is there to be filled
3017 * in, we fill it in now.
3018 */
3019 if (sock->type == SOCK_PACKET) {
Steffen Hurrle342dfc32014-01-17 22:53:15 +01003020 __sockaddr_check_size(sizeof(struct sockaddr_pkt));
Hannes Frederic Sowaf3d33422013-11-21 03:14:22 +01003021 msg->msg_namelen = sizeof(struct sockaddr_pkt);
3022 } else {
3023 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
Eyal Birger2472d762015-03-01 14:58:28 +02003024
Hannes Frederic Sowaf3d33422013-11-21 03:14:22 +01003025 msg->msg_namelen = sll->sll_halen +
3026 offsetof(struct sockaddr_ll, sll_addr);
3027 }
Herbert Xuffbc6112007-02-04 23:33:10 -08003028 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
3029 msg->msg_namelen);
Hannes Frederic Sowaf3d33422013-11-21 03:14:22 +01003030 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003031
Herbert Xu8dc41942007-02-04 23:31:32 -08003032 if (pkt_sk(sk)->auxdata) {
Herbert Xuffbc6112007-02-04 23:33:10 -08003033 struct tpacket_auxdata aux;
3034
3035 aux.tp_status = TP_STATUS_USER;
3036 if (skb->ip_summed == CHECKSUM_PARTIAL)
3037 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
Alexander Drozdov682f0482015-03-23 09:11:13 +03003038 else if (skb->pkt_type != PACKET_OUTGOING &&
3039 (skb->ip_summed == CHECKSUM_COMPLETE ||
3040 skb_csum_unnecessary(skb)))
3041 aux.tp_status |= TP_STATUS_CSUM_VALID;
3042
Eyal Birger2472d762015-03-01 14:58:28 +02003043 aux.tp_len = origlen;
Herbert Xuffbc6112007-02-04 23:33:10 -08003044 aux.tp_snaplen = skb->len;
3045 aux.tp_mac = 0;
Arnaldo Carvalho de Melobbe735e2007-03-10 22:16:10 -03003046 aux.tp_net = skb_network_offset(skb);
Jiri Pirkodf8a39d2015-01-13 17:13:44 +01003047 if (skb_vlan_tag_present(skb)) {
3048 aux.tp_vlan_tci = skb_vlan_tag_get(skb);
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +09003049 aux.tp_vlan_tpid = ntohs(skb->vlan_proto);
3050 aux.tp_status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
Ben Greeara3bcc232011-06-01 06:49:10 +00003051 } else {
3052 aux.tp_vlan_tci = 0;
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +09003053 aux.tp_vlan_tpid = 0;
Ben Greeara3bcc232011-06-01 06:49:10 +00003054 }
Herbert Xuffbc6112007-02-04 23:33:10 -08003055 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
Herbert Xu8dc41942007-02-04 23:31:32 -08003056 }
3057
Linus Torvalds1da177e2005-04-16 15:20:36 -07003058 /*
3059 * Free or return the buffer as appropriate. Again this
3060 * hides all the races and re-entrancy issues from us.
3061 */
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003062 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003063
3064out_free:
3065 skb_free_datagram(sk, skb);
3066out:
3067 return err;
3068}
3069
Linus Torvalds1da177e2005-04-16 15:20:36 -07003070static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
3071 int *uaddr_len, int peer)
3072{
3073 struct net_device *dev;
3074 struct sock *sk = sock->sk;
3075
3076 if (peer)
3077 return -EOPNOTSUPP;
3078
3079 uaddr->sa_family = AF_PACKET;
Daniel Borkmann2dc85bf2013-06-12 16:02:27 +02003080 memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data));
Eric Dumazet654d1f82009-11-02 10:43:32 +01003081 rcu_read_lock();
3082 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
3083 if (dev)
Daniel Borkmann2dc85bf2013-06-12 16:02:27 +02003084 strlcpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data));
Eric Dumazet654d1f82009-11-02 10:43:32 +01003085 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003086 *uaddr_len = sizeof(*uaddr);
3087
3088 return 0;
3089}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003090
3091static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
3092 int *uaddr_len, int peer)
3093{
3094 struct net_device *dev;
3095 struct sock *sk = sock->sk;
3096 struct packet_sock *po = pkt_sk(sk);
Cyrill Gorcunov13cfa972009-11-08 05:51:19 +00003097 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003098
3099 if (peer)
3100 return -EOPNOTSUPP;
3101
3102 sll->sll_family = AF_PACKET;
3103 sll->sll_ifindex = po->ifindex;
3104 sll->sll_protocol = po->num;
Vasiliy Kulikov67286642010-11-10 12:09:10 -08003105 sll->sll_pkttype = 0;
Eric Dumazet654d1f82009-11-02 10:43:32 +01003106 rcu_read_lock();
3107 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003108 if (dev) {
3109 sll->sll_hatype = dev->type;
3110 sll->sll_halen = dev->addr_len;
3111 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003112 } else {
3113 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
3114 sll->sll_halen = 0;
3115 }
Eric Dumazet654d1f82009-11-02 10:43:32 +01003116 rcu_read_unlock();
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07003117 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003118
3119 return 0;
3120}
3121
Wang Chen2aeb0b82008-07-14 20:49:46 -07003122static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
3123 int what)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003124{
3125 switch (i->type) {
3126 case PACKET_MR_MULTICAST:
Jiri Pirko11625632010-03-02 20:40:01 +00003127 if (i->alen != dev->addr_len)
3128 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003129 if (what > 0)
Jiri Pirko22bedad32010-04-01 21:22:57 +00003130 return dev_mc_add(dev, i->addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003131 else
Jiri Pirko22bedad32010-04-01 21:22:57 +00003132 return dev_mc_del(dev, i->addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003133 break;
3134 case PACKET_MR_PROMISC:
Wang Chen2aeb0b82008-07-14 20:49:46 -07003135 return dev_set_promiscuity(dev, what);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003136 case PACKET_MR_ALLMULTI:
Wang Chen2aeb0b82008-07-14 20:49:46 -07003137 return dev_set_allmulti(dev, what);
Eric W. Biedermand95ed922009-05-19 18:27:17 +00003138 case PACKET_MR_UNICAST:
Jiri Pirko11625632010-03-02 20:40:01 +00003139 if (i->alen != dev->addr_len)
3140 return -EINVAL;
Eric W. Biedermand95ed922009-05-19 18:27:17 +00003141 if (what > 0)
Jiri Pirkoa748ee22010-04-01 21:22:09 +00003142 return dev_uc_add(dev, i->addr);
Eric W. Biedermand95ed922009-05-19 18:27:17 +00003143 else
Jiri Pirkoa748ee22010-04-01 21:22:09 +00003144 return dev_uc_del(dev, i->addr);
Eric W. Biedermand95ed922009-05-19 18:27:17 +00003145 break;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003146 default:
3147 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003148 }
Wang Chen2aeb0b82008-07-14 20:49:46 -07003149 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003150}
3151
Francesco Ruggeri82f17092015-03-09 11:51:04 -07003152static void packet_dev_mclist_delete(struct net_device *dev,
3153 struct packet_mclist **mlp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003154{
Francesco Ruggeri82f17092015-03-09 11:51:04 -07003155 struct packet_mclist *ml;
3156
3157 while ((ml = *mlp) != NULL) {
3158 if (ml->ifindex == dev->ifindex) {
3159 packet_dev_mc(dev, ml, -1);
3160 *mlp = ml->next;
3161 kfree(ml);
3162 } else
3163 mlp = &ml->next;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003164 }
3165}
3166
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07003167static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003168{
3169 struct packet_sock *po = pkt_sk(sk);
3170 struct packet_mclist *ml, *i;
3171 struct net_device *dev;
3172 int err;
3173
3174 rtnl_lock();
3175
3176 err = -ENODEV;
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09003177 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003178 if (!dev)
3179 goto done;
3180
3181 err = -EINVAL;
Jiri Pirko11625632010-03-02 20:40:01 +00003182 if (mreq->mr_alen > dev->addr_len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003183 goto done;
3184
3185 err = -ENOBUFS;
Kris Katterjohn8b3a7002006-01-11 15:56:43 -08003186 i = kmalloc(sizeof(*i), GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003187 if (i == NULL)
3188 goto done;
3189
3190 err = 0;
3191 for (ml = po->mclist; ml; ml = ml->next) {
3192 if (ml->ifindex == mreq->mr_ifindex &&
3193 ml->type == mreq->mr_type &&
3194 ml->alen == mreq->mr_alen &&
3195 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3196 ml->count++;
3197 /* Free the new element ... */
3198 kfree(i);
3199 goto done;
3200 }
3201 }
3202
3203 i->type = mreq->mr_type;
3204 i->ifindex = mreq->mr_ifindex;
3205 i->alen = mreq->mr_alen;
3206 memcpy(i->addr, mreq->mr_address, i->alen);
3207 i->count = 1;
3208 i->next = po->mclist;
3209 po->mclist = i;
Wang Chen2aeb0b82008-07-14 20:49:46 -07003210 err = packet_dev_mc(dev, i, 1);
3211 if (err) {
3212 po->mclist = i->next;
3213 kfree(i);
3214 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003215
3216done:
3217 rtnl_unlock();
3218 return err;
3219}
3220
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07003221static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003222{
3223 struct packet_mclist *ml, **mlp;
3224
3225 rtnl_lock();
3226
3227 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
3228 if (ml->ifindex == mreq->mr_ifindex &&
3229 ml->type == mreq->mr_type &&
3230 ml->alen == mreq->mr_alen &&
3231 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3232 if (--ml->count == 0) {
3233 struct net_device *dev;
3234 *mlp = ml->next;
Eric Dumazetad959e72009-10-16 06:38:46 +00003235 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3236 if (dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003237 packet_dev_mc(dev, ml, -1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003238 kfree(ml);
3239 }
Francesco Ruggeri82f17092015-03-09 11:51:04 -07003240 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003241 }
3242 }
3243 rtnl_unlock();
Francesco Ruggeri82f17092015-03-09 11:51:04 -07003244 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003245}
3246
3247static void packet_flush_mclist(struct sock *sk)
3248{
3249 struct packet_sock *po = pkt_sk(sk);
3250 struct packet_mclist *ml;
3251
3252 if (!po->mclist)
3253 return;
3254
3255 rtnl_lock();
3256 while ((ml = po->mclist) != NULL) {
3257 struct net_device *dev;
3258
3259 po->mclist = ml->next;
Eric Dumazetad959e72009-10-16 06:38:46 +00003260 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3261 if (dev != NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003262 packet_dev_mc(dev, ml, -1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003263 kfree(ml);
3264 }
3265 rtnl_unlock();
3266}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003267
3268static int
David S. Millerb7058842009-09-30 16:12:20 -07003269packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003270{
3271 struct sock *sk = sock->sk;
Herbert Xu8dc41942007-02-04 23:31:32 -08003272 struct packet_sock *po = pkt_sk(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003273 int ret;
3274
3275 if (level != SOL_PACKET)
3276 return -ENOPROTOOPT;
3277
Johann Baudy69e3c752009-05-18 22:11:22 -07003278 switch (optname) {
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003279 case PACKET_ADD_MEMBERSHIP:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003280 case PACKET_DROP_MEMBERSHIP:
3281 {
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07003282 struct packet_mreq_max mreq;
3283 int len = optlen;
3284 memset(&mreq, 0, sizeof(mreq));
3285 if (len < sizeof(struct packet_mreq))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003286 return -EINVAL;
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07003287 if (len > sizeof(mreq))
3288 len = sizeof(mreq);
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003289 if (copy_from_user(&mreq, optval, len))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003290 return -EFAULT;
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07003291 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3292 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003293 if (optname == PACKET_ADD_MEMBERSHIP)
3294 ret = packet_mc_add(sk, &mreq);
3295 else
3296 ret = packet_mc_drop(sk, &mreq);
3297 return ret;
3298 }
David S. Millera2efcfa2007-05-29 13:12:50 -07003299
Linus Torvalds1da177e2005-04-16 15:20:36 -07003300 case PACKET_RX_RING:
Johann Baudy69e3c752009-05-18 22:11:22 -07003301 case PACKET_TX_RING:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003302 {
chetan lokef6fb8f102011-08-19 10:18:16 +00003303 union tpacket_req_u req_u;
3304 int len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003305
chetan lokef6fb8f102011-08-19 10:18:16 +00003306 switch (po->tp_version) {
3307 case TPACKET_V1:
3308 case TPACKET_V2:
3309 len = sizeof(req_u.req);
3310 break;
3311 case TPACKET_V3:
3312 default:
3313 len = sizeof(req_u.req3);
3314 break;
3315 }
3316 if (optlen < len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003317 return -EINVAL;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003318 if (pkt_sk(sk)->has_vnet_hdr)
3319 return -EINVAL;
chetan lokef6fb8f102011-08-19 10:18:16 +00003320 if (copy_from_user(&req_u.req, optval, len))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003321 return -EFAULT;
chetan lokef6fb8f102011-08-19 10:18:16 +00003322 return packet_set_ring(sk, &req_u, 0,
3323 optname == PACKET_TX_RING);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003324 }
3325 case PACKET_COPY_THRESH:
3326 {
3327 int val;
3328
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003329 if (optlen != sizeof(val))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003330 return -EINVAL;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003331 if (copy_from_user(&val, optval, sizeof(val)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003332 return -EFAULT;
3333
3334 pkt_sk(sk)->copy_thresh = val;
3335 return 0;
3336 }
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003337 case PACKET_VERSION:
3338 {
3339 int val;
3340
3341 if (optlen != sizeof(val))
3342 return -EINVAL;
Johann Baudy69e3c752009-05-18 22:11:22 -07003343 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003344 return -EBUSY;
3345 if (copy_from_user(&val, optval, sizeof(val)))
3346 return -EFAULT;
3347 switch (val) {
3348 case TPACKET_V1:
3349 case TPACKET_V2:
chetan lokef6fb8f102011-08-19 10:18:16 +00003350 case TPACKET_V3:
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003351 po->tp_version = val;
3352 return 0;
3353 default:
3354 return -EINVAL;
3355 }
3356 }
Patrick McHardy8913336a2008-07-18 18:05:19 -07003357 case PACKET_RESERVE:
3358 {
3359 unsigned int val;
3360
3361 if (optlen != sizeof(val))
3362 return -EINVAL;
Johann Baudy69e3c752009-05-18 22:11:22 -07003363 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
Patrick McHardy8913336a2008-07-18 18:05:19 -07003364 return -EBUSY;
3365 if (copy_from_user(&val, optval, sizeof(val)))
3366 return -EFAULT;
3367 po->tp_reserve = val;
3368 return 0;
3369 }
Johann Baudy69e3c752009-05-18 22:11:22 -07003370 case PACKET_LOSS:
3371 {
3372 unsigned int val;
3373
3374 if (optlen != sizeof(val))
3375 return -EINVAL;
3376 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3377 return -EBUSY;
3378 if (copy_from_user(&val, optval, sizeof(val)))
3379 return -EFAULT;
3380 po->tp_loss = !!val;
3381 return 0;
3382 }
Herbert Xu8dc41942007-02-04 23:31:32 -08003383 case PACKET_AUXDATA:
3384 {
3385 int val;
3386
3387 if (optlen < sizeof(val))
3388 return -EINVAL;
3389 if (copy_from_user(&val, optval, sizeof(val)))
3390 return -EFAULT;
3391
3392 po->auxdata = !!val;
3393 return 0;
3394 }
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07003395 case PACKET_ORIGDEV:
3396 {
3397 int val;
3398
3399 if (optlen < sizeof(val))
3400 return -EINVAL;
3401 if (copy_from_user(&val, optval, sizeof(val)))
3402 return -EFAULT;
3403
3404 po->origdev = !!val;
3405 return 0;
3406 }
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003407 case PACKET_VNET_HDR:
3408 {
3409 int val;
3410
3411 if (sock->type != SOCK_RAW)
3412 return -EINVAL;
3413 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3414 return -EBUSY;
3415 if (optlen < sizeof(val))
3416 return -EINVAL;
3417 if (copy_from_user(&val, optval, sizeof(val)))
3418 return -EFAULT;
3419
3420 po->has_vnet_hdr = !!val;
3421 return 0;
3422 }
Scott McMillan614f60f2010-06-02 05:53:56 -07003423 case PACKET_TIMESTAMP:
3424 {
3425 int val;
3426
3427 if (optlen != sizeof(val))
3428 return -EINVAL;
3429 if (copy_from_user(&val, optval, sizeof(val)))
3430 return -EFAULT;
3431
3432 po->tp_tstamp = val;
3433 return 0;
3434 }
David S. Millerdc99f602011-07-05 01:45:05 -07003435 case PACKET_FANOUT:
3436 {
3437 int val;
3438
3439 if (optlen != sizeof(val))
3440 return -EINVAL;
3441 if (copy_from_user(&val, optval, sizeof(val)))
3442 return -EFAULT;
3443
3444 return fanout_add(sk, val & 0xffff, val >> 16);
3445 }
Paul Chavent5920cd3a2012-11-06 23:10:47 +00003446 case PACKET_TX_HAS_OFF:
3447 {
3448 unsigned int val;
3449
3450 if (optlen != sizeof(val))
3451 return -EINVAL;
3452 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3453 return -EBUSY;
3454 if (copy_from_user(&val, optval, sizeof(val)))
3455 return -EFAULT;
3456 po->tp_tx_has_off = !!val;
3457 return 0;
3458 }
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01003459 case PACKET_QDISC_BYPASS:
3460 {
3461 int val;
3462
3463 if (optlen != sizeof(val))
3464 return -EINVAL;
3465 if (copy_from_user(&val, optval, sizeof(val)))
3466 return -EFAULT;
3467
3468 po->xmit = val ? packet_direct_xmit : dev_queue_xmit;
3469 return 0;
3470 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003471 default:
3472 return -ENOPROTOOPT;
3473 }
3474}
3475
3476static int packet_getsockopt(struct socket *sock, int level, int optname,
3477 char __user *optval, int __user *optlen)
3478{
3479 int len;
Eric Dumazetc06fff62012-04-19 21:56:11 +00003480 int val, lv = sizeof(val);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003481 struct sock *sk = sock->sk;
3482 struct packet_sock *po = pkt_sk(sk);
Eric Dumazetc06fff62012-04-19 21:56:11 +00003483 void *data = &val;
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00003484 union tpacket_stats_u st;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003485
3486 if (level != SOL_PACKET)
3487 return -ENOPROTOOPT;
3488
Kris Katterjohn8ae55f02006-01-23 16:28:02 -08003489 if (get_user(len, optlen))
3490 return -EFAULT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003491
3492 if (len < 0)
3493 return -EINVAL;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003494
Johann Baudy69e3c752009-05-18 22:11:22 -07003495 switch (optname) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003496 case PACKET_STATISTICS:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003497 spin_lock_bh(&sk->sk_receive_queue.lock);
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00003498 memcpy(&st, &po->stats, sizeof(st));
3499 memset(&po->stats, 0, sizeof(po->stats));
3500 spin_unlock_bh(&sk->sk_receive_queue.lock);
3501
chetan lokef6fb8f102011-08-19 10:18:16 +00003502 if (po->tp_version == TPACKET_V3) {
Eric Dumazetc06fff62012-04-19 21:56:11 +00003503 lv = sizeof(struct tpacket_stats_v3);
Willem de Bruijn8bcdeaf2013-08-19 16:40:22 -04003504 st.stats3.tp_packets += st.stats3.tp_drops;
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00003505 data = &st.stats3;
chetan lokef6fb8f102011-08-19 10:18:16 +00003506 } else {
Eric Dumazetc06fff62012-04-19 21:56:11 +00003507 lv = sizeof(struct tpacket_stats);
Willem de Bruijn8bcdeaf2013-08-19 16:40:22 -04003508 st.stats1.tp_packets += st.stats1.tp_drops;
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00003509 data = &st.stats1;
chetan lokef6fb8f102011-08-19 10:18:16 +00003510 }
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00003511
Linus Torvalds1da177e2005-04-16 15:20:36 -07003512 break;
Herbert Xu8dc41942007-02-04 23:31:32 -08003513 case PACKET_AUXDATA:
Herbert Xu8dc41942007-02-04 23:31:32 -08003514 val = po->auxdata;
Herbert Xu8dc41942007-02-04 23:31:32 -08003515 break;
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07003516 case PACKET_ORIGDEV:
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07003517 val = po->origdev;
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07003518 break;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003519 case PACKET_VNET_HDR:
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003520 val = po->has_vnet_hdr;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003521 break;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003522 case PACKET_VERSION:
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003523 val = po->tp_version;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003524 break;
3525 case PACKET_HDRLEN:
3526 if (len > sizeof(int))
3527 len = sizeof(int);
3528 if (copy_from_user(&val, optval, len))
3529 return -EFAULT;
3530 switch (val) {
3531 case TPACKET_V1:
3532 val = sizeof(struct tpacket_hdr);
3533 break;
3534 case TPACKET_V2:
3535 val = sizeof(struct tpacket2_hdr);
3536 break;
chetan lokef6fb8f102011-08-19 10:18:16 +00003537 case TPACKET_V3:
3538 val = sizeof(struct tpacket3_hdr);
3539 break;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003540 default:
3541 return -EINVAL;
3542 }
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003543 break;
Patrick McHardy8913336a2008-07-18 18:05:19 -07003544 case PACKET_RESERVE:
Patrick McHardy8913336a2008-07-18 18:05:19 -07003545 val = po->tp_reserve;
Patrick McHardy8913336a2008-07-18 18:05:19 -07003546 break;
Johann Baudy69e3c752009-05-18 22:11:22 -07003547 case PACKET_LOSS:
Johann Baudy69e3c752009-05-18 22:11:22 -07003548 val = po->tp_loss;
Johann Baudy69e3c752009-05-18 22:11:22 -07003549 break;
Scott McMillan614f60f2010-06-02 05:53:56 -07003550 case PACKET_TIMESTAMP:
Scott McMillan614f60f2010-06-02 05:53:56 -07003551 val = po->tp_tstamp;
Scott McMillan614f60f2010-06-02 05:53:56 -07003552 break;
David S. Millerdc99f602011-07-05 01:45:05 -07003553 case PACKET_FANOUT:
David S. Millerdc99f602011-07-05 01:45:05 -07003554 val = (po->fanout ?
3555 ((u32)po->fanout->id |
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00003556 ((u32)po->fanout->type << 16) |
3557 ((u32)po->fanout->flags << 24)) :
David S. Millerdc99f602011-07-05 01:45:05 -07003558 0);
David S. Millerdc99f602011-07-05 01:45:05 -07003559 break;
Paul Chavent5920cd3a2012-11-06 23:10:47 +00003560 case PACKET_TX_HAS_OFF:
3561 val = po->tp_tx_has_off;
3562 break;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01003563 case PACKET_QDISC_BYPASS:
3564 val = packet_use_direct_xmit(po);
3565 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003566 default:
3567 return -ENOPROTOOPT;
3568 }
3569
Eric Dumazetc06fff62012-04-19 21:56:11 +00003570 if (len > lv)
3571 len = lv;
Kris Katterjohn8ae55f02006-01-23 16:28:02 -08003572 if (put_user(len, optlen))
3573 return -EFAULT;
Herbert Xu8dc41942007-02-04 23:31:32 -08003574 if (copy_to_user(optval, data, len))
3575 return -EFAULT;
Kris Katterjohn8ae55f02006-01-23 16:28:02 -08003576 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003577}
3578
3579
Jiri Pirko351638e2013-05-28 01:30:21 +00003580static int packet_notifier(struct notifier_block *this,
3581 unsigned long msg, void *ptr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003582{
3583 struct sock *sk;
Jiri Pirko351638e2013-05-28 01:30:21 +00003584 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09003585 struct net *net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003586
stephen hemminger808f5112010-02-22 07:57:18 +00003587 rcu_read_lock();
Sasha Levinb67bfe02013-02-27 17:06:00 -08003588 sk_for_each_rcu(sk, &net->packet.sklist) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003589 struct packet_sock *po = pkt_sk(sk);
3590
3591 switch (msg) {
3592 case NETDEV_UNREGISTER:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003593 if (po->mclist)
Francesco Ruggeri82f17092015-03-09 11:51:04 -07003594 packet_dev_mclist_delete(dev, &po->mclist);
David S. Millera2efcfa2007-05-29 13:12:50 -07003595 /* fallthrough */
3596
Linus Torvalds1da177e2005-04-16 15:20:36 -07003597 case NETDEV_DOWN:
3598 if (dev->ifindex == po->ifindex) {
3599 spin_lock(&po->bind_lock);
3600 if (po->running) {
David S. Millerce06b032011-07-04 01:44:29 -07003601 __unregister_prot_hook(sk, false);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003602 sk->sk_err = ENETDOWN;
3603 if (!sock_flag(sk, SOCK_DEAD))
3604 sk->sk_error_report(sk);
3605 }
3606 if (msg == NETDEV_UNREGISTER) {
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01003607 packet_cached_dev_reset(po);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003608 po->ifindex = -1;
Ben Greear160ff182011-06-01 07:18:52 +00003609 if (po->prot_hook.dev)
3610 dev_put(po->prot_hook.dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003611 po->prot_hook.dev = NULL;
3612 }
3613 spin_unlock(&po->bind_lock);
3614 }
3615 break;
3616 case NETDEV_UP:
stephen hemminger808f5112010-02-22 07:57:18 +00003617 if (dev->ifindex == po->ifindex) {
3618 spin_lock(&po->bind_lock);
David S. Millerce06b032011-07-04 01:44:29 -07003619 if (po->num)
3620 register_prot_hook(sk);
stephen hemminger808f5112010-02-22 07:57:18 +00003621 spin_unlock(&po->bind_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003622 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003623 break;
3624 }
3625 }
stephen hemminger808f5112010-02-22 07:57:18 +00003626 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003627 return NOTIFY_DONE;
3628}
3629
3630
3631static int packet_ioctl(struct socket *sock, unsigned int cmd,
3632 unsigned long arg)
3633{
3634 struct sock *sk = sock->sk;
3635
Johann Baudy69e3c752009-05-18 22:11:22 -07003636 switch (cmd) {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003637 case SIOCOUTQ:
3638 {
3639 int amount = sk_wmem_alloc_get(sk);
Eric Dumazet31e6d362009-06-17 19:05:41 -07003640
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003641 return put_user(amount, (int __user *)arg);
3642 }
3643 case SIOCINQ:
3644 {
3645 struct sk_buff *skb;
3646 int amount = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003647
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003648 spin_lock_bh(&sk->sk_receive_queue.lock);
3649 skb = skb_peek(&sk->sk_receive_queue);
3650 if (skb)
3651 amount = skb->len;
3652 spin_unlock_bh(&sk->sk_receive_queue.lock);
3653 return put_user(amount, (int __user *)arg);
3654 }
3655 case SIOCGSTAMP:
3656 return sock_get_timestamp(sk, (struct timeval __user *)arg);
3657 case SIOCGSTAMPNS:
3658 return sock_get_timestampns(sk, (struct timespec __user *)arg);
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003659
Linus Torvalds1da177e2005-04-16 15:20:36 -07003660#ifdef CONFIG_INET
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003661 case SIOCADDRT:
3662 case SIOCDELRT:
3663 case SIOCDARP:
3664 case SIOCGARP:
3665 case SIOCSARP:
3666 case SIOCGIFADDR:
3667 case SIOCSIFADDR:
3668 case SIOCGIFBRDADDR:
3669 case SIOCSIFBRDADDR:
3670 case SIOCGIFNETMASK:
3671 case SIOCSIFNETMASK:
3672 case SIOCGIFDSTADDR:
3673 case SIOCSIFDSTADDR:
3674 case SIOCSIFFLAGS:
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003675 return inet_dgram_ops.ioctl(sock, cmd, arg);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003676#endif
3677
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003678 default:
3679 return -ENOIOCTLCMD;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003680 }
3681 return 0;
3682}
3683
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003684static unsigned int packet_poll(struct file *file, struct socket *sock,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003685 poll_table *wait)
3686{
3687 struct sock *sk = sock->sk;
3688 struct packet_sock *po = pkt_sk(sk);
3689 unsigned int mask = datagram_poll(file, sock, wait);
3690
3691 spin_lock_bh(&sk->sk_receive_queue.lock);
Johann Baudy69e3c752009-05-18 22:11:22 -07003692 if (po->rx_ring.pg_vec) {
chetan lokef6fb8f102011-08-19 10:18:16 +00003693 if (!packet_previous_rx_frame(po, &po->rx_ring,
3694 TP_STATUS_KERNEL))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003695 mask |= POLLIN | POLLRDNORM;
3696 }
3697 spin_unlock_bh(&sk->sk_receive_queue.lock);
Johann Baudy69e3c752009-05-18 22:11:22 -07003698 spin_lock_bh(&sk->sk_write_queue.lock);
3699 if (po->tx_ring.pg_vec) {
3700 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
3701 mask |= POLLOUT | POLLWRNORM;
3702 }
3703 spin_unlock_bh(&sk->sk_write_queue.lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003704 return mask;
3705}
3706
3707
3708/* Dirty? Well, I still did not learn better way to account
3709 * for user mmaps.
3710 */
3711
3712static void packet_mm_open(struct vm_area_struct *vma)
3713{
3714 struct file *file = vma->vm_file;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003715 struct socket *sock = file->private_data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003716 struct sock *sk = sock->sk;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003717
Linus Torvalds1da177e2005-04-16 15:20:36 -07003718 if (sk)
3719 atomic_inc(&pkt_sk(sk)->mapped);
3720}
3721
3722static void packet_mm_close(struct vm_area_struct *vma)
3723{
3724 struct file *file = vma->vm_file;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003725 struct socket *sock = file->private_data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003726 struct sock *sk = sock->sk;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003727
Linus Torvalds1da177e2005-04-16 15:20:36 -07003728 if (sk)
3729 atomic_dec(&pkt_sk(sk)->mapped);
3730}
3731
Alexey Dobriyanf0f37e22009-09-27 22:29:37 +04003732static const struct vm_operations_struct packet_mmap_ops = {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003733 .open = packet_mm_open,
3734 .close = packet_mm_close,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003735};
3736
Neil Horman0e3125c2010-11-16 10:26:47 -08003737static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
3738 unsigned int len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003739{
3740 int i;
3741
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003742 for (i = 0; i < len; i++) {
Neil Horman0e3125c2010-11-16 10:26:47 -08003743 if (likely(pg_vec[i].buffer)) {
Changli Gaoc56b4d92010-12-01 02:52:57 +00003744 if (is_vmalloc_addr(pg_vec[i].buffer))
Neil Horman0e3125c2010-11-16 10:26:47 -08003745 vfree(pg_vec[i].buffer);
3746 else
3747 free_pages((unsigned long)pg_vec[i].buffer,
3748 order);
3749 pg_vec[i].buffer = NULL;
3750 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003751 }
3752 kfree(pg_vec);
3753}
3754
Olof Johanssoneea49cc92011-11-02 11:00:49 +00003755static char *alloc_one_pg_vec_page(unsigned long order)
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003756{
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01003757 char *buffer;
Neil Horman0e3125c2010-11-16 10:26:47 -08003758 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
3759 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
Eric Dumazet719bfea2009-04-15 03:39:52 -07003760
Neil Horman0e3125c2010-11-16 10:26:47 -08003761 buffer = (char *) __get_free_pages(gfp_flags, order);
Neil Horman0e3125c2010-11-16 10:26:47 -08003762 if (buffer)
3763 return buffer;
3764
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01003765 /* __get_free_pages failed, fall back to vmalloc */
Eric Dumazetbbce5a52010-11-20 07:31:54 +00003766 buffer = vzalloc((1 << order) * PAGE_SIZE);
Neil Horman0e3125c2010-11-16 10:26:47 -08003767 if (buffer)
3768 return buffer;
3769
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01003770 /* vmalloc failed, lets dig into swap here */
Neil Horman0e3125c2010-11-16 10:26:47 -08003771 gfp_flags &= ~__GFP_NORETRY;
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01003772 buffer = (char *) __get_free_pages(gfp_flags, order);
Neil Horman0e3125c2010-11-16 10:26:47 -08003773 if (buffer)
3774 return buffer;
3775
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01003776 /* complete and utter failure */
Neil Horman0e3125c2010-11-16 10:26:47 -08003777 return NULL;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003778}
3779
Neil Horman0e3125c2010-11-16 10:26:47 -08003780static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003781{
3782 unsigned int block_nr = req->tp_block_nr;
Neil Horman0e3125c2010-11-16 10:26:47 -08003783 struct pgv *pg_vec;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003784 int i;
3785
Neil Horman0e3125c2010-11-16 10:26:47 -08003786 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003787 if (unlikely(!pg_vec))
3788 goto out;
3789
3790 for (i = 0; i < block_nr; i++) {
Changli Gaoc56b4d92010-12-01 02:52:57 +00003791 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
Neil Horman0e3125c2010-11-16 10:26:47 -08003792 if (unlikely(!pg_vec[i].buffer))
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003793 goto out_free_pgvec;
3794 }
3795
3796out:
3797 return pg_vec;
3798
3799out_free_pgvec:
3800 free_pg_vec(pg_vec, order, block_nr);
3801 pg_vec = NULL;
3802 goto out;
3803}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003804
chetan lokef6fb8f102011-08-19 10:18:16 +00003805static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
Johann Baudy69e3c752009-05-18 22:11:22 -07003806 int closing, int tx_ring)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003807{
Neil Horman0e3125c2010-11-16 10:26:47 -08003808 struct pgv *pg_vec = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003809 struct packet_sock *po = pkt_sk(sk);
Al Viro0e11c912006-11-08 00:26:29 -08003810 int was_running, order = 0;
Johann Baudy69e3c752009-05-18 22:11:22 -07003811 struct packet_ring_buffer *rb;
3812 struct sk_buff_head *rb_queue;
Al Viro0e11c912006-11-08 00:26:29 -08003813 __be16 num;
chetan lokef6fb8f102011-08-19 10:18:16 +00003814 int err = -EINVAL;
3815 /* Added to avoid minimal code churn */
3816 struct tpacket_req *req = &req_u->req;
3817
3818 /* Opening a Tx-ring is NOT supported in TPACKET_V3 */
3819 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) {
3820 WARN(1, "Tx-ring is not supported.\n");
3821 goto out;
3822 }
Johann Baudy69e3c752009-05-18 22:11:22 -07003823
3824 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
3825 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
3826
3827 err = -EBUSY;
3828 if (!closing) {
3829 if (atomic_read(&po->mapped))
3830 goto out;
Daniel Borkmannb0138402014-01-15 16:25:36 +01003831 if (packet_read_pending(rb))
Johann Baudy69e3c752009-05-18 22:11:22 -07003832 goto out;
3833 }
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003834
Linus Torvalds1da177e2005-04-16 15:20:36 -07003835 if (req->tp_block_nr) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003836 /* Sanity tests and some calculations */
Johann Baudy69e3c752009-05-18 22:11:22 -07003837 err = -EBUSY;
3838 if (unlikely(rb->pg_vec))
3839 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003840
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003841 switch (po->tp_version) {
3842 case TPACKET_V1:
3843 po->tp_hdrlen = TPACKET_HDRLEN;
3844 break;
3845 case TPACKET_V2:
3846 po->tp_hdrlen = TPACKET2_HDRLEN;
3847 break;
chetan lokef6fb8f102011-08-19 10:18:16 +00003848 case TPACKET_V3:
3849 po->tp_hdrlen = TPACKET3_HDRLEN;
3850 break;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003851 }
3852
Johann Baudy69e3c752009-05-18 22:11:22 -07003853 err = -EINVAL;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003854 if (unlikely((int)req->tp_block_size <= 0))
Johann Baudy69e3c752009-05-18 22:11:22 -07003855 goto out;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003856 if (unlikely(req->tp_block_size & (PAGE_SIZE - 1)))
Johann Baudy69e3c752009-05-18 22:11:22 -07003857 goto out;
Eric Dumazetdc808112014-08-15 09:16:04 -07003858 if (po->tp_version >= TPACKET_V3 &&
3859 (int)(req->tp_block_size -
3860 BLK_PLUS_PRIV(req_u->req3.tp_sizeof_priv)) <= 0)
3861 goto out;
Patrick McHardy8913336a2008-07-18 18:05:19 -07003862 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
Johann Baudy69e3c752009-05-18 22:11:22 -07003863 po->tp_reserve))
3864 goto out;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003865 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
Johann Baudy69e3c752009-05-18 22:11:22 -07003866 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003867
Johann Baudy69e3c752009-05-18 22:11:22 -07003868 rb->frames_per_block = req->tp_block_size/req->tp_frame_size;
3869 if (unlikely(rb->frames_per_block <= 0))
3870 goto out;
3871 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
3872 req->tp_frame_nr))
3873 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003874
3875 err = -ENOMEM;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003876 order = get_order(req->tp_block_size);
3877 pg_vec = alloc_pg_vec(req, order);
3878 if (unlikely(!pg_vec))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003879 goto out;
chetan lokef6fb8f102011-08-19 10:18:16 +00003880 switch (po->tp_version) {
3881 case TPACKET_V3:
3882 /* Transmit path is not supported. We checked
3883 * it above but just being paranoid
3884 */
3885 if (!tx_ring)
3886 init_prb_bdqc(po, rb, pg_vec, req_u, tx_ring);
Dan Carpenterd7cf0c32014-02-18 15:20:51 +03003887 break;
chetan lokef6fb8f102011-08-19 10:18:16 +00003888 default:
3889 break;
3890 }
Johann Baudy69e3c752009-05-18 22:11:22 -07003891 }
3892 /* Done */
3893 else {
3894 err = -EINVAL;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003895 if (unlikely(req->tp_frame_nr))
Johann Baudy69e3c752009-05-18 22:11:22 -07003896 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003897 }
3898
3899 lock_sock(sk);
3900
3901 /* Detach socket from network */
3902 spin_lock(&po->bind_lock);
3903 was_running = po->running;
3904 num = po->num;
3905 if (was_running) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003906 po->num = 0;
David S. Millerce06b032011-07-04 01:44:29 -07003907 __unregister_prot_hook(sk, false);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003908 }
3909 spin_unlock(&po->bind_lock);
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003910
Linus Torvalds1da177e2005-04-16 15:20:36 -07003911 synchronize_net();
3912
3913 err = -EBUSY;
Herbert Xu905db442009-01-30 14:12:06 -08003914 mutex_lock(&po->pg_vec_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003915 if (closing || atomic_read(&po->mapped) == 0) {
3916 err = 0;
Johann Baudy69e3c752009-05-18 22:11:22 -07003917 spin_lock_bh(&rb_queue->lock);
Changli Gaoc053fd92010-12-10 16:02:20 -08003918 swap(rb->pg_vec, pg_vec);
Johann Baudy69e3c752009-05-18 22:11:22 -07003919 rb->frame_max = (req->tp_frame_nr - 1);
3920 rb->head = 0;
3921 rb->frame_size = req->tp_frame_size;
3922 spin_unlock_bh(&rb_queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003923
Changli Gaoc053fd92010-12-10 16:02:20 -08003924 swap(rb->pg_vec_order, order);
3925 swap(rb->pg_vec_len, req->tp_block_nr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003926
Johann Baudy69e3c752009-05-18 22:11:22 -07003927 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
3928 po->prot_hook.func = (po->rx_ring.pg_vec) ?
3929 tpacket_rcv : packet_rcv;
3930 skb_queue_purge(rb_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003931 if (atomic_read(&po->mapped))
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003932 pr_err("packet_mmap: vma is busy: %d\n",
3933 atomic_read(&po->mapped));
Linus Torvalds1da177e2005-04-16 15:20:36 -07003934 }
Herbert Xu905db442009-01-30 14:12:06 -08003935 mutex_unlock(&po->pg_vec_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003936
3937 spin_lock(&po->bind_lock);
David S. Millerce06b032011-07-04 01:44:29 -07003938 if (was_running) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003939 po->num = num;
David S. Millerce06b032011-07-04 01:44:29 -07003940 register_prot_hook(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003941 }
3942 spin_unlock(&po->bind_lock);
chetan lokef6fb8f102011-08-19 10:18:16 +00003943 if (closing && (po->tp_version > TPACKET_V2)) {
3944 /* Because we don't support block-based V3 on tx-ring */
3945 if (!tx_ring)
3946 prb_shutdown_retire_blk_timer(po, tx_ring, rb_queue);
3947 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003948 release_sock(sk);
3949
Linus Torvalds1da177e2005-04-16 15:20:36 -07003950 if (pg_vec)
3951 free_pg_vec(pg_vec, order, req->tp_block_nr);
3952out:
3953 return err;
3954}
3955
Johann Baudy69e3c752009-05-18 22:11:22 -07003956static int packet_mmap(struct file *file, struct socket *sock,
3957 struct vm_area_struct *vma)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003958{
3959 struct sock *sk = sock->sk;
3960 struct packet_sock *po = pkt_sk(sk);
Johann Baudy69e3c752009-05-18 22:11:22 -07003961 unsigned long size, expected_size;
3962 struct packet_ring_buffer *rb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003963 unsigned long start;
3964 int err = -EINVAL;
3965 int i;
3966
3967 if (vma->vm_pgoff)
3968 return -EINVAL;
3969
Herbert Xu905db442009-01-30 14:12:06 -08003970 mutex_lock(&po->pg_vec_lock);
Johann Baudy69e3c752009-05-18 22:11:22 -07003971
3972 expected_size = 0;
3973 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3974 if (rb->pg_vec) {
3975 expected_size += rb->pg_vec_len
3976 * rb->pg_vec_pages
3977 * PAGE_SIZE;
3978 }
3979 }
3980
3981 if (expected_size == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003982 goto out;
Johann Baudy69e3c752009-05-18 22:11:22 -07003983
3984 size = vma->vm_end - vma->vm_start;
3985 if (size != expected_size)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003986 goto out;
3987
Linus Torvalds1da177e2005-04-16 15:20:36 -07003988 start = vma->vm_start;
Johann Baudy69e3c752009-05-18 22:11:22 -07003989 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3990 if (rb->pg_vec == NULL)
3991 continue;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003992
Johann Baudy69e3c752009-05-18 22:11:22 -07003993 for (i = 0; i < rb->pg_vec_len; i++) {
Neil Horman0e3125c2010-11-16 10:26:47 -08003994 struct page *page;
3995 void *kaddr = rb->pg_vec[i].buffer;
Johann Baudy69e3c752009-05-18 22:11:22 -07003996 int pg_num;
3997
Changli Gaoc56b4d92010-12-01 02:52:57 +00003998 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
3999 page = pgv_to_page(kaddr);
Johann Baudy69e3c752009-05-18 22:11:22 -07004000 err = vm_insert_page(vma, start, page);
4001 if (unlikely(err))
4002 goto out;
4003 start += PAGE_SIZE;
Neil Horman0e3125c2010-11-16 10:26:47 -08004004 kaddr += PAGE_SIZE;
Johann Baudy69e3c752009-05-18 22:11:22 -07004005 }
David S. Miller4ebf0ae2005-12-06 16:38:35 -08004006 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004007 }
Johann Baudy69e3c752009-05-18 22:11:22 -07004008
David S. Miller4ebf0ae2005-12-06 16:38:35 -08004009 atomic_inc(&po->mapped);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004010 vma->vm_ops = &packet_mmap_ops;
4011 err = 0;
4012
4013out:
Herbert Xu905db442009-01-30 14:12:06 -08004014 mutex_unlock(&po->pg_vec_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004015 return err;
4016}
Linus Torvalds1da177e2005-04-16 15:20:36 -07004017
Eric Dumazet90ddc4f2005-12-22 12:49:22 -08004018static const struct proto_ops packet_ops_spkt = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004019 .family = PF_PACKET,
4020 .owner = THIS_MODULE,
4021 .release = packet_release,
4022 .bind = packet_bind_spkt,
4023 .connect = sock_no_connect,
4024 .socketpair = sock_no_socketpair,
4025 .accept = sock_no_accept,
4026 .getname = packet_getname_spkt,
4027 .poll = datagram_poll,
4028 .ioctl = packet_ioctl,
4029 .listen = sock_no_listen,
4030 .shutdown = sock_no_shutdown,
4031 .setsockopt = sock_no_setsockopt,
4032 .getsockopt = sock_no_getsockopt,
4033 .sendmsg = packet_sendmsg_spkt,
4034 .recvmsg = packet_recvmsg,
4035 .mmap = sock_no_mmap,
4036 .sendpage = sock_no_sendpage,
4037};
Linus Torvalds1da177e2005-04-16 15:20:36 -07004038
Eric Dumazet90ddc4f2005-12-22 12:49:22 -08004039static const struct proto_ops packet_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004040 .family = PF_PACKET,
4041 .owner = THIS_MODULE,
4042 .release = packet_release,
4043 .bind = packet_bind,
4044 .connect = sock_no_connect,
4045 .socketpair = sock_no_socketpair,
4046 .accept = sock_no_accept,
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09004047 .getname = packet_getname,
Linus Torvalds1da177e2005-04-16 15:20:36 -07004048 .poll = packet_poll,
4049 .ioctl = packet_ioctl,
4050 .listen = sock_no_listen,
4051 .shutdown = sock_no_shutdown,
4052 .setsockopt = packet_setsockopt,
4053 .getsockopt = packet_getsockopt,
4054 .sendmsg = packet_sendmsg,
4055 .recvmsg = packet_recvmsg,
4056 .mmap = packet_mmap,
4057 .sendpage = sock_no_sendpage,
4058};
4059
Stephen Hemmingerec1b4cf2009-10-05 05:58:39 +00004060static const struct net_proto_family packet_family_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004061 .family = PF_PACKET,
4062 .create = packet_create,
4063 .owner = THIS_MODULE,
4064};
4065
4066static struct notifier_block packet_netdev_notifier = {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00004067 .notifier_call = packet_notifier,
Linus Torvalds1da177e2005-04-16 15:20:36 -07004068};
4069
4070#ifdef CONFIG_PROC_FS
Linus Torvalds1da177e2005-04-16 15:20:36 -07004071
4072static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
stephen hemminger808f5112010-02-22 07:57:18 +00004073 __acquires(RCU)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004074{
Denis V. Luneve372c412007-11-19 22:31:54 -08004075 struct net *net = seq_file_net(seq);
stephen hemminger808f5112010-02-22 07:57:18 +00004076
4077 rcu_read_lock();
4078 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004079}
4080
4081static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4082{
Herbert Xu1bf40952007-12-16 14:04:02 -08004083 struct net *net = seq_file_net(seq);
stephen hemminger808f5112010-02-22 07:57:18 +00004084 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004085}
4086
4087static void packet_seq_stop(struct seq_file *seq, void *v)
stephen hemminger808f5112010-02-22 07:57:18 +00004088 __releases(RCU)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004089{
stephen hemminger808f5112010-02-22 07:57:18 +00004090 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004091}
4092
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09004093static int packet_seq_show(struct seq_file *seq, void *v)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004094{
4095 if (v == SEQ_START_TOKEN)
4096 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
4097 else {
Li Zefanb7ceabd2010-02-08 23:19:29 +00004098 struct sock *s = sk_entry(v);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004099 const struct packet_sock *po = pkt_sk(s);
4100
4101 seq_printf(seq,
Dan Rosenberg71338aa2011-05-23 12:17:35 +00004102 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07004103 s,
4104 atomic_read(&s->sk_refcnt),
4105 s->sk_type,
4106 ntohs(po->num),
4107 po->ifindex,
4108 po->running,
4109 atomic_read(&s->sk_rmem_alloc),
Eric W. Biedermana7cb5a42012-05-24 01:10:10 -06004110 from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00004111 sock_i_ino(s));
Linus Torvalds1da177e2005-04-16 15:20:36 -07004112 }
4113
4114 return 0;
4115}
4116
Philippe De Muyter56b3d972007-07-10 23:07:31 -07004117static const struct seq_operations packet_seq_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004118 .start = packet_seq_start,
4119 .next = packet_seq_next,
4120 .stop = packet_seq_stop,
4121 .show = packet_seq_show,
4122};
4123
4124static int packet_seq_open(struct inode *inode, struct file *file)
4125{
Denis V. Luneve372c412007-11-19 22:31:54 -08004126 return seq_open_net(inode, file, &packet_seq_ops,
4127 sizeof(struct seq_net_private));
Linus Torvalds1da177e2005-04-16 15:20:36 -07004128}
4129
Arjan van de Venda7071d2007-02-12 00:55:36 -08004130static const struct file_operations packet_seq_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004131 .owner = THIS_MODULE,
4132 .open = packet_seq_open,
4133 .read = seq_read,
4134 .llseek = seq_lseek,
Denis V. Luneve372c412007-11-19 22:31:54 -08004135 .release = seq_release_net,
Linus Torvalds1da177e2005-04-16 15:20:36 -07004136};
4137
4138#endif
4139
Alexey Dobriyan2c8c1e72010-01-17 03:35:32 +00004140static int __net_init packet_net_init(struct net *net)
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004141{
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00004142 mutex_init(&net->packet.sklist_lock);
Denis V. Lunev2aaef4e2007-12-11 04:19:54 -08004143 INIT_HLIST_HEAD(&net->packet.sklist);
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004144
Gao fengd4beaa62013-02-18 01:34:54 +00004145 if (!proc_create("packet", 0, net->proc_net, &packet_seq_fops))
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004146 return -ENOMEM;
4147
4148 return 0;
4149}
4150
Alexey Dobriyan2c8c1e72010-01-17 03:35:32 +00004151static void __net_exit packet_net_exit(struct net *net)
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004152{
Gao fengece31ff2013-02-18 01:34:56 +00004153 remove_proc_entry("packet", net->proc_net);
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004154}
4155
4156static struct pernet_operations packet_net_ops = {
4157 .init = packet_net_init,
4158 .exit = packet_net_exit,
4159};
4160
4161
Linus Torvalds1da177e2005-04-16 15:20:36 -07004162static void __exit packet_exit(void)
4163{
Linus Torvalds1da177e2005-04-16 15:20:36 -07004164 unregister_netdevice_notifier(&packet_netdev_notifier);
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004165 unregister_pernet_subsys(&packet_net_ops);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004166 sock_unregister(PF_PACKET);
4167 proto_unregister(&packet_proto);
4168}
4169
4170static int __init packet_init(void)
4171{
4172 int rc = proto_register(&packet_proto, 0);
4173
4174 if (rc != 0)
4175 goto out;
4176
4177 sock_register(&packet_family_ops);
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004178 register_pernet_subsys(&packet_net_ops);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004179 register_netdevice_notifier(&packet_netdev_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004180out:
4181 return rc;
4182}
4183
4184module_init(packet_init);
4185module_exit(packet_exit);
4186MODULE_LICENSE("GPL");
4187MODULE_ALIAS_NETPROTO(PF_PACKET);