blob: 8298e676f5a015f58d1b6005cf85938f8c8e142a [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * PACKET - implements raw packet sockets.
7 *
Jesper Juhl02c30a82005-05-05 16:16:16 -07008 * Authors: Ross Biro
Linus Torvalds1da177e2005-04-16 15:20:36 -07009 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
11 *
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +090012 * Fixes:
Linus Torvalds1da177e2005-04-16 15:20:36 -070013 * Alan Cox : verify_area() now used correctly
14 * Alan Cox : new skbuff lists, look ma no backlogs!
15 * Alan Cox : tidied skbuff lists.
16 * Alan Cox : Now uses generic datagram routines I
17 * added. Also fixed the peek/read crash
18 * from all old Linux datagram code.
19 * Alan Cox : Uses the improved datagram code.
20 * Alan Cox : Added NULL's for socket options.
21 * Alan Cox : Re-commented the code.
22 * Alan Cox : Use new kernel side addressing
23 * Rob Janssen : Correct MTU usage.
24 * Dave Platt : Counter leaks caused by incorrect
25 * interrupt locking and some slightly
26 * dubious gcc output. Can you read
27 * compiler: it said _VOLATILE_
28 * Richard Kooijman : Timestamp fixes.
29 * Alan Cox : New buffers. Use sk->mac.raw.
30 * Alan Cox : sendmsg/recvmsg support.
31 * Alan Cox : Protocol setting support
32 * Alexey Kuznetsov : Untied from IPv4 stack.
33 * Cyrus Durgin : Fixed kerneld for kmod.
34 * Michal Ostrowski : Module initialization cleanup.
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +090035 * Ulises Alonso : Frame number limit removal and
Linus Torvalds1da177e2005-04-16 15:20:36 -070036 * packet_set_ring memory leak.
Eric W. Biederman0fb375f2005-09-21 00:11:37 -070037 * Eric Biederman : Allow for > 8 byte hardware addresses.
38 * The convention is that longer addresses
39 * will simply extend the hardware address
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +090040 * byte arrays at the end of sockaddr_ll
Eric W. Biederman0fb375f2005-09-21 00:11:37 -070041 * and packet_mreq.
Johann Baudy69e3c752009-05-18 22:11:22 -070042 * Johann Baudy : Added TX RING.
Linus Torvalds1da177e2005-04-16 15:20:36 -070043 *
44 * This program is free software; you can redistribute it and/or
45 * modify it under the terms of the GNU General Public License
46 * as published by the Free Software Foundation; either version
47 * 2 of the License, or (at your option) any later version.
48 *
49 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +090050
Linus Torvalds1da177e2005-04-16 15:20:36 -070051#include <linux/types.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070052#include <linux/mm.h>
Randy Dunlap4fc268d2006-01-11 12:17:47 -080053#include <linux/capability.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070054#include <linux/fcntl.h>
55#include <linux/socket.h>
56#include <linux/in.h>
57#include <linux/inet.h>
58#include <linux/netdevice.h>
59#include <linux/if_packet.h>
60#include <linux/wireless.h>
Herbert Xuffbc6112007-02-04 23:33:10 -080061#include <linux/kernel.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070062#include <linux/kmod.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090063#include <linux/slab.h>
Eric W. Biederman457c4cb2007-09-12 12:01:34 +020064#include <net/net_namespace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070065#include <net/ip.h>
66#include <net/protocol.h>
67#include <linux/skbuff.h>
68#include <net/sock.h>
69#include <linux/errno.h>
70#include <linux/timer.h>
71#include <asm/system.h>
72#include <asm/uaccess.h>
73#include <asm/ioctls.h>
74#include <asm/page.h>
Al Viroa1f8e7f2006-10-19 16:08:53 -040075#include <asm/cacheflush.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070076#include <asm/io.h>
77#include <linux/proc_fs.h>
78#include <linux/seq_file.h>
79#include <linux/poll.h>
80#include <linux/module.h>
81#include <linux/init.h>
Herbert Xu905db442009-01-30 14:12:06 -080082#include <linux/mutex.h>
Eric Dumazet05423b22009-10-26 18:40:35 -070083#include <linux/if_vlan.h>
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -080084#include <linux/virtio_net.h>
Richard Cochraned85b562010-04-07 22:41:28 +000085#include <linux/errqueue.h>
Scott McMillan614f60f2010-06-02 05:53:56 -070086#include <linux/net_tstamp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070087
88#ifdef CONFIG_INET
89#include <net/inet_common.h>
90#endif
91
Linus Torvalds1da177e2005-04-16 15:20:36 -070092/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070093 Assumptions:
94 - if device has no dev->hard_header routine, it adds and removes ll header
95 inside itself. In this case ll header is invisible outside of device,
96 but higher levels still should reserve dev->hard_header_len.
97 Some devices are enough clever to reallocate skb, when header
98 will not fit to reserved space (tunnel), another ones are silly
99 (PPP).
100 - packet socket receives packets with pulled ll header,
101 so that SOCK_RAW should push it back.
102
103On receive:
104-----------
105
106Incoming, dev->hard_header!=NULL
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -0700107 mac_header -> ll header
108 data -> data
Linus Torvalds1da177e2005-04-16 15:20:36 -0700109
110Outgoing, dev->hard_header!=NULL
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -0700111 mac_header -> ll header
112 data -> ll header
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113
114Incoming, dev->hard_header==NULL
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -0700115 mac_header -> UNKNOWN position. It is very likely, that it points to ll
116 header. PPP makes it, that is wrong, because introduce
YOSHIFUJI Hideakidb0c58f2007-07-19 10:44:35 +0900117 assymetry between rx and tx paths.
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -0700118 data -> data
Linus Torvalds1da177e2005-04-16 15:20:36 -0700119
120Outgoing, dev->hard_header==NULL
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -0700121 mac_header -> data. ll header is still not built!
122 data -> data
Linus Torvalds1da177e2005-04-16 15:20:36 -0700123
124Resume
125 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
126
127
128On transmit:
129------------
130
131dev->hard_header != NULL
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -0700132 mac_header -> ll header
133 data -> ll header
Linus Torvalds1da177e2005-04-16 15:20:36 -0700134
135dev->hard_header == NULL (ll header is added by device, we cannot control it)
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -0700136 mac_header -> data
137 data -> data
Linus Torvalds1da177e2005-04-16 15:20:36 -0700138
139 We should set nh.raw on output to correct posistion,
140 packet classifier depends on it.
141 */
142
Linus Torvalds1da177e2005-04-16 15:20:36 -0700143/* Private packet socket structures. */
144
Eric Dumazet40d4e3d2009-07-21 21:57:59 +0000145struct packet_mclist {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700146 struct packet_mclist *next;
147 int ifindex;
148 int count;
149 unsigned short type;
150 unsigned short alen;
Eric W. Biederman0fb375f2005-09-21 00:11:37 -0700151 unsigned char addr[MAX_ADDR_LEN];
152};
153/* identical to struct packet_mreq except it has
154 * a longer address field.
155 */
Eric Dumazet40d4e3d2009-07-21 21:57:59 +0000156struct packet_mreq_max {
Eric W. Biederman0fb375f2005-09-21 00:11:37 -0700157 int mr_ifindex;
158 unsigned short mr_type;
159 unsigned short mr_alen;
160 unsigned char mr_address[MAX_ADDR_LEN];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700161};
David S. Millera2efcfa2007-05-29 13:12:50 -0700162
Johann Baudy69e3c752009-05-18 22:11:22 -0700163static int packet_set_ring(struct sock *sk, struct tpacket_req *req,
164 int closing, int tx_ring);
165
166struct packet_ring_buffer {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +0000167 char **pg_vec;
Johann Baudy69e3c752009-05-18 22:11:22 -0700168 unsigned int head;
169 unsigned int frames_per_block;
170 unsigned int frame_size;
171 unsigned int frame_max;
172
173 unsigned int pg_vec_order;
174 unsigned int pg_vec_pages;
175 unsigned int pg_vec_len;
176
177 atomic_t pending;
178};
179
180struct packet_sock;
181static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700182
183static void packet_flush_mclist(struct sock *sk);
184
185struct packet_sock {
186 /* struct sock has to be the first member of packet_sock */
187 struct sock sk;
188 struct tpacket_stats stats;
Johann Baudy69e3c752009-05-18 22:11:22 -0700189 struct packet_ring_buffer rx_ring;
190 struct packet_ring_buffer tx_ring;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700191 int copy_thresh;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700192 spinlock_t bind_lock;
Herbert Xu905db442009-01-30 14:12:06 -0800193 struct mutex pg_vec_lock;
Herbert Xu8dc41942007-02-04 23:31:32 -0800194 unsigned int running:1, /* prot_hook is attached*/
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -0700195 auxdata:1,
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -0800196 origdev:1,
197 has_vnet_hdr:1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700198 int ifindex; /* bound device */
Al Viro0e11c912006-11-08 00:26:29 -0800199 __be16 num;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700200 struct packet_mclist *mclist;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700201 atomic_t mapped;
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700202 enum tpacket_versions tp_version;
203 unsigned int tp_hdrlen;
Patrick McHardy89133362008-07-18 18:05:19 -0700204 unsigned int tp_reserve;
Johann Baudy69e3c752009-05-18 22:11:22 -0700205 unsigned int tp_loss:1;
Scott McMillan614f60f2010-06-02 05:53:56 -0700206 unsigned int tp_tstamp;
Eric Dumazet94b05952009-10-16 04:02:20 +0000207 struct packet_type prot_hook ____cacheline_aligned_in_smp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700208};
209
Herbert Xuffbc6112007-02-04 23:33:10 -0800210struct packet_skb_cb {
211 unsigned int origlen;
212 union {
213 struct sockaddr_pkt pkt;
214 struct sockaddr_ll ll;
215 } sa;
216};
217
218#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
Herbert Xu8dc41942007-02-04 23:31:32 -0800219
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700220static void __packet_set_status(struct packet_sock *po, void *frame, int status)
221{
222 union {
223 struct tpacket_hdr *h1;
224 struct tpacket2_hdr *h2;
225 void *raw;
226 } h;
227
228 h.raw = frame;
229 switch (po->tp_version) {
230 case TPACKET_V1:
231 h.h1->tp_status = status;
Johann Baudy69e3c752009-05-18 22:11:22 -0700232 flush_dcache_page(virt_to_page(&h.h1->tp_status));
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700233 break;
234 case TPACKET_V2:
235 h.h2->tp_status = status;
Johann Baudy69e3c752009-05-18 22:11:22 -0700236 flush_dcache_page(virt_to_page(&h.h2->tp_status));
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700237 break;
Johann Baudy69e3c752009-05-18 22:11:22 -0700238 default:
Eric Dumazet40d4e3d2009-07-21 21:57:59 +0000239 pr_err("TPACKET version not supported\n");
Johann Baudy69e3c752009-05-18 22:11:22 -0700240 BUG();
241 }
242
243 smp_wmb();
244}
245
246static int __packet_get_status(struct packet_sock *po, void *frame)
247{
248 union {
249 struct tpacket_hdr *h1;
250 struct tpacket2_hdr *h2;
251 void *raw;
252 } h;
253
254 smp_rmb();
255
256 h.raw = frame;
257 switch (po->tp_version) {
258 case TPACKET_V1:
259 flush_dcache_page(virt_to_page(&h.h1->tp_status));
260 return h.h1->tp_status;
261 case TPACKET_V2:
262 flush_dcache_page(virt_to_page(&h.h2->tp_status));
263 return h.h2->tp_status;
264 default:
Eric Dumazet40d4e3d2009-07-21 21:57:59 +0000265 pr_err("TPACKET version not supported\n");
Johann Baudy69e3c752009-05-18 22:11:22 -0700266 BUG();
267 return 0;
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700268 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700269}
Johann Baudy69e3c752009-05-18 22:11:22 -0700270
271static void *packet_lookup_frame(struct packet_sock *po,
272 struct packet_ring_buffer *rb,
273 unsigned int position,
274 int status)
275{
276 unsigned int pg_vec_pos, frame_offset;
277 union {
278 struct tpacket_hdr *h1;
279 struct tpacket2_hdr *h2;
280 void *raw;
281 } h;
282
283 pg_vec_pos = position / rb->frames_per_block;
284 frame_offset = position % rb->frames_per_block;
285
286 h.raw = rb->pg_vec[pg_vec_pos] + (frame_offset * rb->frame_size);
287
288 if (status != __packet_get_status(po, h.raw))
289 return NULL;
290
291 return h.raw;
292}
293
294static inline void *packet_current_frame(struct packet_sock *po,
295 struct packet_ring_buffer *rb,
296 int status)
297{
298 return packet_lookup_frame(po, rb, rb->head, status);
299}
300
301static inline void *packet_previous_frame(struct packet_sock *po,
302 struct packet_ring_buffer *rb,
303 int status)
304{
305 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
306 return packet_lookup_frame(po, rb, previous, status);
307}
308
309static inline void packet_increment_head(struct packet_ring_buffer *buff)
310{
311 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
312}
313
Linus Torvalds1da177e2005-04-16 15:20:36 -0700314static inline struct packet_sock *pkt_sk(struct sock *sk)
315{
316 return (struct packet_sock *)sk;
317}
318
319static void packet_sock_destruct(struct sock *sk)
320{
Richard Cochraned85b562010-04-07 22:41:28 +0000321 skb_queue_purge(&sk->sk_error_queue);
322
Ilpo Järvinen547b7922008-07-25 21:43:18 -0700323 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
324 WARN_ON(atomic_read(&sk->sk_wmem_alloc));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700325
326 if (!sock_flag(sk, SOCK_DEAD)) {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +0000327 pr_err("Attempt to release alive packet socket: %p\n", sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700328 return;
329 }
330
Pavel Emelyanov17ab56a2007-11-10 21:38:48 -0800331 sk_refcnt_debug_dec(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700332}
333
334
Eric Dumazet90ddc4f2005-12-22 12:49:22 -0800335static const struct proto_ops packet_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700336
Eric Dumazet90ddc4f2005-12-22 12:49:22 -0800337static const struct proto_ops packet_ops_spkt;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700338
Eric Dumazet40d4e3d2009-07-21 21:57:59 +0000339static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
340 struct packet_type *pt, struct net_device *orig_dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700341{
342 struct sock *sk;
343 struct sockaddr_pkt *spkt;
344
345 /*
346 * When we registered the protocol we saved the socket in the data
347 * field for just this event.
348 */
349
350 sk = pt->af_packet_priv;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +0900351
Linus Torvalds1da177e2005-04-16 15:20:36 -0700352 /*
353 * Yank back the headers [hope the device set this
354 * right or kerboom...]
355 *
356 * Incoming packets have ll header pulled,
357 * push it back.
358 *
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -0700359 * For outgoing ones skb->data == skb_mac_header(skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700360 * so that this procedure is noop.
361 */
362
363 if (skb->pkt_type == PACKET_LOOPBACK)
364 goto out;
365
Octavian Purdila09ad9bc2009-11-25 15:14:13 -0800366 if (!net_eq(dev_net(dev), sock_net(sk)))
Denis V. Lunevd12d01d2007-11-19 22:28:35 -0800367 goto out;
368
Eric Dumazet40d4e3d2009-07-21 21:57:59 +0000369 skb = skb_share_check(skb, GFP_ATOMIC);
370 if (skb == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700371 goto oom;
372
373 /* drop any routing info */
Eric Dumazetadf30902009-06-02 05:19:30 +0000374 skb_dst_drop(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700375
Phil Oester84531c22005-07-12 11:57:52 -0700376 /* drop conntrack reference */
377 nf_reset(skb);
378
Herbert Xuffbc6112007-02-04 23:33:10 -0800379 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700380
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -0700381 skb_push(skb, skb->data - skb_mac_header(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700382
383 /*
384 * The SOCK_PACKET socket receives _all_ frames.
385 */
386
387 spkt->spkt_family = dev->type;
388 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
389 spkt->spkt_protocol = skb->protocol;
390
391 /*
392 * Charge the memory to the socket. This is done specifically
393 * to prevent sockets using all the memory up.
394 */
395
Eric Dumazet40d4e3d2009-07-21 21:57:59 +0000396 if (sock_queue_rcv_skb(sk, skb) == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700397 return 0;
398
399out:
400 kfree_skb(skb);
401oom:
402 return 0;
403}
404
405
406/*
407 * Output a raw packet to a device layer. This bypasses all the other
408 * protocol layers and you must therefore supply it with a complete frame
409 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +0900410
Linus Torvalds1da177e2005-04-16 15:20:36 -0700411static int packet_sendmsg_spkt(struct kiocb *iocb, struct socket *sock,
412 struct msghdr *msg, size_t len)
413{
414 struct sock *sk = sock->sk;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +0000415 struct sockaddr_pkt *saddr = (struct sockaddr_pkt *)msg->msg_name;
Eric Dumazet1a35ca82009-12-15 05:47:03 +0000416 struct sk_buff *skb = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700417 struct net_device *dev;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +0000418 __be16 proto = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700419 int err;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +0900420
Linus Torvalds1da177e2005-04-16 15:20:36 -0700421 /*
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +0900422 * Get and verify the address.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700423 */
424
Eric Dumazet40d4e3d2009-07-21 21:57:59 +0000425 if (saddr) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700426 if (msg->msg_namelen < sizeof(struct sockaddr))
Eric Dumazet40d4e3d2009-07-21 21:57:59 +0000427 return -EINVAL;
428 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
429 proto = saddr->spkt_protocol;
430 } else
431 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700432
433 /*
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +0900434 * Find the device first to size check it
Linus Torvalds1da177e2005-04-16 15:20:36 -0700435 */
436
437 saddr->spkt_device[13] = 0;
Eric Dumazet1a35ca82009-12-15 05:47:03 +0000438retry:
Eric Dumazet654d1f82009-11-02 10:43:32 +0100439 rcu_read_lock();
440 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700441 err = -ENODEV;
442 if (dev == NULL)
443 goto out_unlock;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +0900444
David S. Millerd5e76b02007-01-25 19:30:36 -0800445 err = -ENETDOWN;
446 if (!(dev->flags & IFF_UP))
447 goto out_unlock;
448
Linus Torvalds1da177e2005-04-16 15:20:36 -0700449 /*
Eric Dumazet40d4e3d2009-07-21 21:57:59 +0000450 * You may not queue a frame bigger than the mtu. This is the lowest level
451 * raw protocol and you must do your own fragmentation at this level.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700452 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +0900453
Linus Torvalds1da177e2005-04-16 15:20:36 -0700454 err = -EMSGSIZE;
Kris Katterjohn8ae55f02006-01-23 16:28:02 -0800455 if (len > dev->mtu + dev->hard_header_len)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700456 goto out_unlock;
457
Eric Dumazet1a35ca82009-12-15 05:47:03 +0000458 if (!skb) {
459 size_t reserved = LL_RESERVED_SPACE(dev);
460 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700461
Eric Dumazet1a35ca82009-12-15 05:47:03 +0000462 rcu_read_unlock();
463 skb = sock_wmalloc(sk, len + reserved, 0, GFP_KERNEL);
464 if (skb == NULL)
465 return -ENOBUFS;
466 /* FIXME: Save some space for broken drivers that write a hard
467 * header at transmission time by themselves. PPP is the notable
468 * one here. This should really be fixed at the driver level.
469 */
470 skb_reserve(skb, reserved);
471 skb_reset_network_header(skb);
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +0900472
Eric Dumazet1a35ca82009-12-15 05:47:03 +0000473 /* Try to align data part correctly */
474 if (hhlen) {
475 skb->data -= hhlen;
476 skb->tail -= hhlen;
477 if (len < hhlen)
478 skb_reset_network_header(skb);
479 }
480 err = memcpy_fromiovec(skb_put(skb, len), msg->msg_iov, len);
481 if (err)
482 goto out_free;
483 goto retry;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700484 }
485
Eric Dumazet1a35ca82009-12-15 05:47:03 +0000486
Linus Torvalds1da177e2005-04-16 15:20:36 -0700487 skb->protocol = proto;
488 skb->dev = dev;
489 skb->priority = sk->sk_priority;
Eric Dumazet2d37a182009-10-01 19:14:46 +0000490 skb->mark = sk->sk_mark;
Oliver Hartkopp2244d072010-08-17 08:59:14 +0000491 err = sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
Richard Cochraned85b562010-04-07 22:41:28 +0000492 if (err < 0)
493 goto out_unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700494
495 dev_queue_xmit(skb);
Eric Dumazet654d1f82009-11-02 10:43:32 +0100496 rcu_read_unlock();
Eric Dumazet40d4e3d2009-07-21 21:57:59 +0000497 return len;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700498
Linus Torvalds1da177e2005-04-16 15:20:36 -0700499out_unlock:
Eric Dumazet654d1f82009-11-02 10:43:32 +0100500 rcu_read_unlock();
Eric Dumazet1a35ca82009-12-15 05:47:03 +0000501out_free:
502 kfree_skb(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700503 return err;
504}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700505
David S. Millerdbcb5852007-01-24 15:21:02 -0800506static inline unsigned int run_filter(struct sk_buff *skb, struct sock *sk,
507 unsigned int res)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700508{
509 struct sk_filter *filter;
510
Dmitry Mishinfda9ef52006-08-31 15:28:39 -0700511 rcu_read_lock_bh();
Paul E. McKenneya898def2010-02-22 17:04:49 -0800512 filter = rcu_dereference_bh(sk->sk_filter);
David S. Millerdbcb5852007-01-24 15:21:02 -0800513 if (filter != NULL)
514 res = sk_run_filter(skb, filter->insns, filter->len);
Dmitry Mishinfda9ef52006-08-31 15:28:39 -0700515 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700516
David S. Millerdbcb5852007-01-24 15:21:02 -0800517 return res;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700518}
519
520/*
521 This function makes lazy skb cloning in hope that most of packets
522 are discarded by BPF.
523
524 Note tricky part: we DO mangle shared skb! skb->data, skb->len
525 and skb->cb are mangled. It works because (and until) packets
526 falling here are owned by current CPU. Output packets are cloned
527 by dev_queue_xmit_nit(), input packets are processed by net_bh
528 sequencially, so that if we return skb to original state on exit,
529 we will not harm anyone.
530 */
531
Eric Dumazet40d4e3d2009-07-21 21:57:59 +0000532static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
533 struct packet_type *pt, struct net_device *orig_dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700534{
535 struct sock *sk;
536 struct sockaddr_ll *sll;
537 struct packet_sock *po;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +0000538 u8 *skb_head = skb->data;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700539 int skb_len = skb->len;
David S. Millerdbcb5852007-01-24 15:21:02 -0800540 unsigned int snaplen, res;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700541
542 if (skb->pkt_type == PACKET_LOOPBACK)
543 goto drop;
544
545 sk = pt->af_packet_priv;
546 po = pkt_sk(sk);
547
Octavian Purdila09ad9bc2009-11-25 15:14:13 -0800548 if (!net_eq(dev_net(dev), sock_net(sk)))
Denis V. Lunevd12d01d2007-11-19 22:28:35 -0800549 goto drop;
550
Linus Torvalds1da177e2005-04-16 15:20:36 -0700551 skb->dev = dev;
552
Stephen Hemminger3b04ddd2007-10-09 01:40:57 -0700553 if (dev->header_ops) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700554 /* The device has an explicit notion of ll header,
555 exported to higher levels.
556
557 Otherwise, the device hides datails of it frame
558 structure, so that corresponding packet head
559 never delivered to user.
560 */
561 if (sk->sk_type != SOCK_DGRAM)
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -0700562 skb_push(skb, skb->data - skb_mac_header(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700563 else if (skb->pkt_type == PACKET_OUTGOING) {
564 /* Special case: outgoing packets have ll header at head */
Arnaldo Carvalho de Melobbe735e2007-03-10 22:16:10 -0300565 skb_pull(skb, skb_network_offset(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700566 }
567 }
568
569 snaplen = skb->len;
570
David S. Millerdbcb5852007-01-24 15:21:02 -0800571 res = run_filter(skb, sk, snaplen);
572 if (!res)
Dmitry Mishinfda9ef52006-08-31 15:28:39 -0700573 goto drop_n_restore;
David S. Millerdbcb5852007-01-24 15:21:02 -0800574 if (snaplen > res)
575 snaplen = res;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700576
577 if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >=
578 (unsigned)sk->sk_rcvbuf)
579 goto drop_n_acct;
580
581 if (skb_shared(skb)) {
582 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
583 if (nskb == NULL)
584 goto drop_n_acct;
585
586 if (skb_head != skb->data) {
587 skb->data = skb_head;
588 skb->len = skb_len;
589 }
590 kfree_skb(skb);
591 skb = nskb;
592 }
593
Herbert Xuffbc6112007-02-04 23:33:10 -0800594 BUILD_BUG_ON(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8 >
595 sizeof(skb->cb));
596
597 sll = &PACKET_SKB_CB(skb)->sa.ll;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700598 sll->sll_family = AF_PACKET;
599 sll->sll_hatype = dev->type;
600 sll->sll_protocol = skb->protocol;
601 sll->sll_pkttype = skb->pkt_type;
Peter P Waskiewicz Jr8032b462007-11-10 22:03:25 -0800602 if (unlikely(po->origdev))
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -0700603 sll->sll_ifindex = orig_dev->ifindex;
604 else
605 sll->sll_ifindex = dev->ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700606
Stephen Hemmingerb95cce32007-09-26 22:13:38 -0700607 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700608
Herbert Xuffbc6112007-02-04 23:33:10 -0800609 PACKET_SKB_CB(skb)->origlen = skb->len;
Herbert Xu8dc41942007-02-04 23:31:32 -0800610
Linus Torvalds1da177e2005-04-16 15:20:36 -0700611 if (pskb_trim(skb, snaplen))
612 goto drop_n_acct;
613
614 skb_set_owner_r(skb, sk);
615 skb->dev = NULL;
Eric Dumazetadf30902009-06-02 05:19:30 +0000616 skb_dst_drop(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700617
Phil Oester84531c22005-07-12 11:57:52 -0700618 /* drop conntrack reference */
619 nf_reset(skb);
620
Linus Torvalds1da177e2005-04-16 15:20:36 -0700621 spin_lock(&sk->sk_receive_queue.lock);
622 po->stats.tp_packets++;
Neil Horman3b885782009-10-12 13:26:31 -0700623 skb->dropcount = atomic_read(&sk->sk_drops);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700624 __skb_queue_tail(&sk->sk_receive_queue, skb);
625 spin_unlock(&sk->sk_receive_queue.lock);
626 sk->sk_data_ready(sk, skb->len);
627 return 0;
628
629drop_n_acct:
Neil Horman3b885782009-10-12 13:26:31 -0700630 po->stats.tp_drops = atomic_inc_return(&sk->sk_drops);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700631
632drop_n_restore:
633 if (skb_head != skb->data && skb_shared(skb)) {
634 skb->data = skb_head;
635 skb->len = skb_len;
636 }
637drop:
Neil Hormanead2ceb2009-03-11 09:49:55 +0000638 consume_skb(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700639 return 0;
640}
641
Eric Dumazet40d4e3d2009-07-21 21:57:59 +0000642static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
643 struct packet_type *pt, struct net_device *orig_dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700644{
645 struct sock *sk;
646 struct packet_sock *po;
647 struct sockaddr_ll *sll;
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700648 union {
649 struct tpacket_hdr *h1;
650 struct tpacket2_hdr *h2;
651 void *raw;
652 } h;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +0000653 u8 *skb_head = skb->data;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700654 int skb_len = skb->len;
David S. Millerdbcb5852007-01-24 15:21:02 -0800655 unsigned int snaplen, res;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700656 unsigned long status = TP_STATUS_LOSING|TP_STATUS_USER;
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700657 unsigned short macoff, netoff, hdrlen;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700658 struct sk_buff *copy_skb = NULL;
Eric Dumazetb7aa0bf2007-04-19 16:16:32 -0700659 struct timeval tv;
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700660 struct timespec ts;
Scott McMillan614f60f2010-06-02 05:53:56 -0700661 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700662
663 if (skb->pkt_type == PACKET_LOOPBACK)
664 goto drop;
665
666 sk = pt->af_packet_priv;
667 po = pkt_sk(sk);
668
Octavian Purdila09ad9bc2009-11-25 15:14:13 -0800669 if (!net_eq(dev_net(dev), sock_net(sk)))
Denis V. Lunevd12d01d2007-11-19 22:28:35 -0800670 goto drop;
671
Stephen Hemminger3b04ddd2007-10-09 01:40:57 -0700672 if (dev->header_ops) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700673 if (sk->sk_type != SOCK_DGRAM)
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -0700674 skb_push(skb, skb->data - skb_mac_header(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700675 else if (skb->pkt_type == PACKET_OUTGOING) {
676 /* Special case: outgoing packets have ll header at head */
Arnaldo Carvalho de Melobbe735e2007-03-10 22:16:10 -0300677 skb_pull(skb, skb_network_offset(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700678 }
679 }
680
Herbert Xu8dc41942007-02-04 23:31:32 -0800681 if (skb->ip_summed == CHECKSUM_PARTIAL)
682 status |= TP_STATUS_CSUMNOTREADY;
683
Linus Torvalds1da177e2005-04-16 15:20:36 -0700684 snaplen = skb->len;
685
David S. Millerdbcb5852007-01-24 15:21:02 -0800686 res = run_filter(skb, sk, snaplen);
687 if (!res)
Dmitry Mishinfda9ef52006-08-31 15:28:39 -0700688 goto drop_n_restore;
David S. Millerdbcb5852007-01-24 15:21:02 -0800689 if (snaplen > res)
690 snaplen = res;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700691
692 if (sk->sk_type == SOCK_DGRAM) {
Patrick McHardy89133362008-07-18 18:05:19 -0700693 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
694 po->tp_reserve;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700695 } else {
Arnaldo Carvalho de Melobbe735e2007-03-10 22:16:10 -0300696 unsigned maclen = skb_network_offset(skb);
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700697 netoff = TPACKET_ALIGN(po->tp_hdrlen +
Patrick McHardy89133362008-07-18 18:05:19 -0700698 (maclen < 16 ? 16 : maclen)) +
699 po->tp_reserve;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700700 macoff = netoff - maclen;
701 }
702
Johann Baudy69e3c752009-05-18 22:11:22 -0700703 if (macoff + snaplen > po->rx_ring.frame_size) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700704 if (po->copy_thresh &&
705 atomic_read(&sk->sk_rmem_alloc) + skb->truesize <
706 (unsigned)sk->sk_rcvbuf) {
707 if (skb_shared(skb)) {
708 copy_skb = skb_clone(skb, GFP_ATOMIC);
709 } else {
710 copy_skb = skb_get(skb);
711 skb_head = skb->data;
712 }
713 if (copy_skb)
714 skb_set_owner_r(copy_skb, sk);
715 }
Johann Baudy69e3c752009-05-18 22:11:22 -0700716 snaplen = po->rx_ring.frame_size - macoff;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700717 if ((int)snaplen < 0)
718 snaplen = 0;
719 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700720
721 spin_lock(&sk->sk_receive_queue.lock);
Johann Baudy69e3c752009-05-18 22:11:22 -0700722 h.raw = packet_current_frame(po, &po->rx_ring, TP_STATUS_KERNEL);
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700723 if (!h.raw)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700724 goto ring_is_full;
Johann Baudy69e3c752009-05-18 22:11:22 -0700725 packet_increment_head(&po->rx_ring);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700726 po->stats.tp_packets++;
727 if (copy_skb) {
728 status |= TP_STATUS_COPY;
729 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
730 }
731 if (!po->stats.tp_drops)
732 status &= ~TP_STATUS_LOSING;
733 spin_unlock(&sk->sk_receive_queue.lock);
734
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700735 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700736
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700737 switch (po->tp_version) {
738 case TPACKET_V1:
739 h.h1->tp_len = skb->len;
740 h.h1->tp_snaplen = snaplen;
741 h.h1->tp_mac = macoff;
742 h.h1->tp_net = netoff;
Scott McMillan614f60f2010-06-02 05:53:56 -0700743 if ((po->tp_tstamp & SOF_TIMESTAMPING_SYS_HARDWARE)
744 && shhwtstamps->syststamp.tv64)
745 tv = ktime_to_timeval(shhwtstamps->syststamp);
746 else if ((po->tp_tstamp & SOF_TIMESTAMPING_RAW_HARDWARE)
747 && shhwtstamps->hwtstamp.tv64)
748 tv = ktime_to_timeval(shhwtstamps->hwtstamp);
749 else if (skb->tstamp.tv64)
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700750 tv = ktime_to_timeval(skb->tstamp);
751 else
752 do_gettimeofday(&tv);
753 h.h1->tp_sec = tv.tv_sec;
754 h.h1->tp_usec = tv.tv_usec;
755 hdrlen = sizeof(*h.h1);
756 break;
757 case TPACKET_V2:
758 h.h2->tp_len = skb->len;
759 h.h2->tp_snaplen = snaplen;
760 h.h2->tp_mac = macoff;
761 h.h2->tp_net = netoff;
Scott McMillan614f60f2010-06-02 05:53:56 -0700762 if ((po->tp_tstamp & SOF_TIMESTAMPING_SYS_HARDWARE)
763 && shhwtstamps->syststamp.tv64)
764 ts = ktime_to_timespec(shhwtstamps->syststamp);
765 else if ((po->tp_tstamp & SOF_TIMESTAMPING_RAW_HARDWARE)
766 && shhwtstamps->hwtstamp.tv64)
767 ts = ktime_to_timespec(shhwtstamps->hwtstamp);
768 else if (skb->tstamp.tv64)
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700769 ts = ktime_to_timespec(skb->tstamp);
770 else
771 getnstimeofday(&ts);
772 h.h2->tp_sec = ts.tv_sec;
773 h.h2->tp_nsec = ts.tv_nsec;
Eric Dumazet05423b22009-10-26 18:40:35 -0700774 h.h2->tp_vlan_tci = vlan_tx_tag_get(skb);
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700775 hdrlen = sizeof(*h.h2);
776 break;
777 default:
778 BUG();
779 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700780
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700781 sll = h.raw + TPACKET_ALIGN(hdrlen);
Stephen Hemmingerb95cce32007-09-26 22:13:38 -0700782 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700783 sll->sll_family = AF_PACKET;
784 sll->sll_hatype = dev->type;
785 sll->sll_protocol = skb->protocol;
786 sll->sll_pkttype = skb->pkt_type;
Peter P Waskiewicz Jr8032b462007-11-10 22:03:25 -0800787 if (unlikely(po->origdev))
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -0700788 sll->sll_ifindex = orig_dev->ifindex;
789 else
790 sll->sll_ifindex = dev->ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700791
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700792 __packet_set_status(po, h.raw, status);
Ralf Baechlee16aa202006-12-07 00:11:33 -0800793 smp_mb();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700794 {
795 struct page *p_start, *p_end;
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700796 u8 *h_end = h.raw + macoff + snaplen - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700797
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700798 p_start = virt_to_page(h.raw);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700799 p_end = virt_to_page(h_end);
800 while (p_start <= p_end) {
801 flush_dcache_page(p_start);
802 p_start++;
803 }
804 }
805
806 sk->sk_data_ready(sk, 0);
807
808drop_n_restore:
809 if (skb_head != skb->data && skb_shared(skb)) {
810 skb->data = skb_head;
811 skb->len = skb_len;
812 }
813drop:
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +0900814 kfree_skb(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700815 return 0;
816
817ring_is_full:
818 po->stats.tp_drops++;
819 spin_unlock(&sk->sk_receive_queue.lock);
820
821 sk->sk_data_ready(sk, 0);
Wei Yongjunacb5d752009-02-25 00:36:42 +0000822 kfree_skb(copy_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700823 goto drop_n_restore;
824}
825
Johann Baudy69e3c752009-05-18 22:11:22 -0700826static void tpacket_destruct_skb(struct sk_buff *skb)
827{
828 struct packet_sock *po = pkt_sk(skb->sk);
Eric Dumazet40d4e3d2009-07-21 21:57:59 +0000829 void *ph;
Johann Baudy69e3c752009-05-18 22:11:22 -0700830
831 BUG_ON(skb == NULL);
832
833 if (likely(po->tx_ring.pg_vec)) {
834 ph = skb_shinfo(skb)->destructor_arg;
835 BUG_ON(__packet_get_status(po, ph) != TP_STATUS_SENDING);
836 BUG_ON(atomic_read(&po->tx_ring.pending) == 0);
837 atomic_dec(&po->tx_ring.pending);
838 __packet_set_status(po, ph, TP_STATUS_AVAILABLE);
839 }
840
841 sock_wfree(skb);
842}
843
Eric Dumazet40d4e3d2009-07-21 21:57:59 +0000844static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
845 void *frame, struct net_device *dev, int size_max,
846 __be16 proto, unsigned char *addr)
Johann Baudy69e3c752009-05-18 22:11:22 -0700847{
848 union {
849 struct tpacket_hdr *h1;
850 struct tpacket2_hdr *h2;
851 void *raw;
852 } ph;
853 int to_write, offset, len, tp_len, nr_frags, len_max;
854 struct socket *sock = po->sk.sk_socket;
855 struct page *page;
856 void *data;
857 int err;
858
859 ph.raw = frame;
860
861 skb->protocol = proto;
862 skb->dev = dev;
863 skb->priority = po->sk.sk_priority;
Eric Dumazet2d37a182009-10-01 19:14:46 +0000864 skb->mark = po->sk.sk_mark;
Johann Baudy69e3c752009-05-18 22:11:22 -0700865 skb_shinfo(skb)->destructor_arg = ph.raw;
866
867 switch (po->tp_version) {
868 case TPACKET_V2:
869 tp_len = ph.h2->tp_len;
870 break;
871 default:
872 tp_len = ph.h1->tp_len;
873 break;
874 }
875 if (unlikely(tp_len > size_max)) {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +0000876 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
Johann Baudy69e3c752009-05-18 22:11:22 -0700877 return -EMSGSIZE;
878 }
879
880 skb_reserve(skb, LL_RESERVED_SPACE(dev));
881 skb_reset_network_header(skb);
882
883 data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll);
884 to_write = tp_len;
885
886 if (sock->type == SOCK_DGRAM) {
887 err = dev_hard_header(skb, dev, ntohs(proto), addr,
888 NULL, tp_len);
889 if (unlikely(err < 0))
890 return -EINVAL;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +0000891 } else if (dev->hard_header_len) {
Johann Baudy69e3c752009-05-18 22:11:22 -0700892 /* net device doesn't like empty head */
893 if (unlikely(tp_len <= dev->hard_header_len)) {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +0000894 pr_err("packet size is too short (%d < %d)\n",
895 tp_len, dev->hard_header_len);
Johann Baudy69e3c752009-05-18 22:11:22 -0700896 return -EINVAL;
897 }
898
899 skb_push(skb, dev->hard_header_len);
900 err = skb_store_bits(skb, 0, data,
901 dev->hard_header_len);
902 if (unlikely(err))
903 return err;
904
905 data += dev->hard_header_len;
906 to_write -= dev->hard_header_len;
907 }
908
909 err = -EFAULT;
910 page = virt_to_page(data);
911 offset = offset_in_page(data);
912 len_max = PAGE_SIZE - offset;
913 len = ((to_write > len_max) ? len_max : to_write);
914
915 skb->data_len = to_write;
916 skb->len += to_write;
917 skb->truesize += to_write;
918 atomic_add(to_write, &po->sk.sk_wmem_alloc);
919
920 while (likely(to_write)) {
921 nr_frags = skb_shinfo(skb)->nr_frags;
922
923 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +0000924 pr_err("Packet exceed the number of skb frags(%lu)\n",
925 MAX_SKB_FRAGS);
Johann Baudy69e3c752009-05-18 22:11:22 -0700926 return -EFAULT;
927 }
928
929 flush_dcache_page(page);
930 get_page(page);
931 skb_fill_page_desc(skb,
932 nr_frags,
933 page++, offset, len);
934 to_write -= len;
935 offset = 0;
936 len_max = PAGE_SIZE;
937 len = ((to_write > len_max) ? len_max : to_write);
938 }
939
940 return tp_len;
941}
942
943static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
944{
945 struct socket *sock;
946 struct sk_buff *skb;
947 struct net_device *dev;
948 __be16 proto;
949 int ifindex, err, reserve = 0;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +0000950 void *ph;
951 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
Johann Baudy69e3c752009-05-18 22:11:22 -0700952 int tp_len, size_max;
953 unsigned char *addr;
954 int len_sum = 0;
955 int status = 0;
956
957 sock = po->sk.sk_socket;
958
959 mutex_lock(&po->pg_vec_lock);
960
961 err = -EBUSY;
962 if (saddr == NULL) {
963 ifindex = po->ifindex;
964 proto = po->num;
965 addr = NULL;
966 } else {
967 err = -EINVAL;
968 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
969 goto out;
970 if (msg->msg_namelen < (saddr->sll_halen
971 + offsetof(struct sockaddr_ll,
972 sll_addr)))
973 goto out;
974 ifindex = saddr->sll_ifindex;
975 proto = saddr->sll_protocol;
976 addr = saddr->sll_addr;
977 }
978
979 dev = dev_get_by_index(sock_net(&po->sk), ifindex);
980 err = -ENXIO;
981 if (unlikely(dev == NULL))
982 goto out;
983
984 reserve = dev->hard_header_len;
985
986 err = -ENETDOWN;
987 if (unlikely(!(dev->flags & IFF_UP)))
988 goto out_put;
989
990 size_max = po->tx_ring.frame_size
Gabor Gombasb5dd8842009-10-29 03:19:11 -0700991 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
Johann Baudy69e3c752009-05-18 22:11:22 -0700992
993 if (size_max > dev->mtu + reserve)
994 size_max = dev->mtu + reserve;
995
996 do {
997 ph = packet_current_frame(po, &po->tx_ring,
998 TP_STATUS_SEND_REQUEST);
999
1000 if (unlikely(ph == NULL)) {
1001 schedule();
1002 continue;
1003 }
1004
1005 status = TP_STATUS_SEND_REQUEST;
1006 skb = sock_alloc_send_skb(&po->sk,
1007 LL_ALLOCATED_SPACE(dev)
1008 + sizeof(struct sockaddr_ll),
1009 0, &err);
1010
1011 if (unlikely(skb == NULL))
1012 goto out_status;
1013
1014 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto,
1015 addr);
1016
1017 if (unlikely(tp_len < 0)) {
1018 if (po->tp_loss) {
1019 __packet_set_status(po, ph,
1020 TP_STATUS_AVAILABLE);
1021 packet_increment_head(&po->tx_ring);
1022 kfree_skb(skb);
1023 continue;
1024 } else {
1025 status = TP_STATUS_WRONG_FORMAT;
1026 err = tp_len;
1027 goto out_status;
1028 }
1029 }
1030
1031 skb->destructor = tpacket_destruct_skb;
1032 __packet_set_status(po, ph, TP_STATUS_SENDING);
1033 atomic_inc(&po->tx_ring.pending);
1034
1035 status = TP_STATUS_SEND_REQUEST;
1036 err = dev_queue_xmit(skb);
Jarek Poplawskieb70df12010-01-10 22:04:19 +00001037 if (unlikely(err > 0)) {
1038 err = net_xmit_errno(err);
1039 if (err && __packet_get_status(po, ph) ==
1040 TP_STATUS_AVAILABLE) {
1041 /* skb was destructed already */
1042 skb = NULL;
1043 goto out_status;
1044 }
1045 /*
1046 * skb was dropped but not destructed yet;
1047 * let's treat it like congestion or err < 0
1048 */
1049 err = 0;
1050 }
Johann Baudy69e3c752009-05-18 22:11:22 -07001051 packet_increment_head(&po->tx_ring);
1052 len_sum += tp_len;
Joe Perchesf64f9e72009-11-29 16:55:45 -08001053 } while (likely((ph != NULL) ||
1054 ((!(msg->msg_flags & MSG_DONTWAIT)) &&
1055 (atomic_read(&po->tx_ring.pending))))
1056 );
Johann Baudy69e3c752009-05-18 22:11:22 -07001057
1058 err = len_sum;
1059 goto out_put;
1060
Johann Baudy69e3c752009-05-18 22:11:22 -07001061out_status:
1062 __packet_set_status(po, ph, status);
1063 kfree_skb(skb);
1064out_put:
1065 dev_put(dev);
1066out:
1067 mutex_unlock(&po->pg_vec_lock);
1068 return err;
1069}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001070
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08001071static inline struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
1072 size_t reserve, size_t len,
1073 size_t linear, int noblock,
1074 int *err)
1075{
1076 struct sk_buff *skb;
1077
1078 /* Under a page? Don't bother with paged skb. */
1079 if (prepad + len < PAGE_SIZE || !linear)
1080 linear = len;
1081
1082 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
1083 err);
1084 if (!skb)
1085 return NULL;
1086
1087 skb_reserve(skb, reserve);
1088 skb_put(skb, linear);
1089 skb->data_len = len - linear;
1090 skb->len += len - linear;
1091
1092 return skb;
1093}
1094
Johann Baudy69e3c752009-05-18 22:11:22 -07001095static int packet_snd(struct socket *sock,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001096 struct msghdr *msg, size_t len)
1097{
1098 struct sock *sk = sock->sk;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001099 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001100 struct sk_buff *skb;
1101 struct net_device *dev;
Al Viro0e11c912006-11-08 00:26:29 -08001102 __be16 proto;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001103 unsigned char *addr;
1104 int ifindex, err, reserve = 0;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08001105 struct virtio_net_hdr vnet_hdr = { 0 };
1106 int offset = 0;
1107 int vnet_hdr_len;
1108 struct packet_sock *po = pkt_sk(sk);
1109 unsigned short gso_type = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001110
1111 /*
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001112 * Get and verify the address.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001113 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001114
Linus Torvalds1da177e2005-04-16 15:20:36 -07001115 if (saddr == NULL) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001116 ifindex = po->ifindex;
1117 proto = po->num;
1118 addr = NULL;
1119 } else {
1120 err = -EINVAL;
1121 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
1122 goto out;
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07001123 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
1124 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001125 ifindex = saddr->sll_ifindex;
1126 proto = saddr->sll_protocol;
1127 addr = saddr->sll_addr;
1128 }
1129
1130
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09001131 dev = dev_get_by_index(sock_net(sk), ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001132 err = -ENXIO;
1133 if (dev == NULL)
1134 goto out_unlock;
1135 if (sock->type == SOCK_RAW)
1136 reserve = dev->hard_header_len;
1137
David S. Millerd5e76b02007-01-25 19:30:36 -08001138 err = -ENETDOWN;
1139 if (!(dev->flags & IFF_UP))
1140 goto out_unlock;
1141
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08001142 if (po->has_vnet_hdr) {
1143 vnet_hdr_len = sizeof(vnet_hdr);
1144
1145 err = -EINVAL;
1146 if (len < vnet_hdr_len)
1147 goto out_unlock;
1148
1149 len -= vnet_hdr_len;
1150
1151 err = memcpy_fromiovec((void *)&vnet_hdr, msg->msg_iov,
1152 vnet_hdr_len);
1153 if (err < 0)
1154 goto out_unlock;
1155
1156 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
1157 (vnet_hdr.csum_start + vnet_hdr.csum_offset + 2 >
1158 vnet_hdr.hdr_len))
1159 vnet_hdr.hdr_len = vnet_hdr.csum_start +
1160 vnet_hdr.csum_offset + 2;
1161
1162 err = -EINVAL;
1163 if (vnet_hdr.hdr_len > len)
1164 goto out_unlock;
1165
1166 if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
1167 switch (vnet_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
1168 case VIRTIO_NET_HDR_GSO_TCPV4:
1169 gso_type = SKB_GSO_TCPV4;
1170 break;
1171 case VIRTIO_NET_HDR_GSO_TCPV6:
1172 gso_type = SKB_GSO_TCPV6;
1173 break;
1174 case VIRTIO_NET_HDR_GSO_UDP:
1175 gso_type = SKB_GSO_UDP;
1176 break;
1177 default:
1178 goto out_unlock;
1179 }
1180
1181 if (vnet_hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
1182 gso_type |= SKB_GSO_TCP_ECN;
1183
1184 if (vnet_hdr.gso_size == 0)
1185 goto out_unlock;
1186
1187 }
1188 }
1189
Linus Torvalds1da177e2005-04-16 15:20:36 -07001190 err = -EMSGSIZE;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08001191 if (!gso_type && (len > dev->mtu+reserve))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001192 goto out_unlock;
1193
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08001194 err = -ENOBUFS;
1195 skb = packet_alloc_skb(sk, LL_ALLOCATED_SPACE(dev),
1196 LL_RESERVED_SPACE(dev), len, vnet_hdr.hdr_len,
1197 msg->msg_flags & MSG_DONTWAIT, &err);
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001198 if (skb == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001199 goto out_unlock;
1200
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08001201 skb_set_network_header(skb, reserve);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001202
Stephen Hemminger0c4e8582007-10-09 01:36:32 -07001203 err = -EINVAL;
1204 if (sock->type == SOCK_DGRAM &&
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08001205 (offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len)) < 0)
Stephen Hemminger0c4e8582007-10-09 01:36:32 -07001206 goto out_free;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001207
1208 /* Returns -EFAULT on error */
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08001209 err = skb_copy_datagram_from_iovec(skb, offset, msg->msg_iov, 0, len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001210 if (err)
1211 goto out_free;
Oliver Hartkopp2244d072010-08-17 08:59:14 +00001212 err = sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
Richard Cochraned85b562010-04-07 22:41:28 +00001213 if (err < 0)
1214 goto out_free;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001215
1216 skb->protocol = proto;
1217 skb->dev = dev;
1218 skb->priority = sk->sk_priority;
Eric Dumazet2d37a182009-10-01 19:14:46 +00001219 skb->mark = sk->sk_mark;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001220
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08001221 if (po->has_vnet_hdr) {
1222 if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
1223 if (!skb_partial_csum_set(skb, vnet_hdr.csum_start,
1224 vnet_hdr.csum_offset)) {
1225 err = -EINVAL;
1226 goto out_free;
1227 }
1228 }
1229
1230 skb_shinfo(skb)->gso_size = vnet_hdr.gso_size;
1231 skb_shinfo(skb)->gso_type = gso_type;
1232
1233 /* Header must be checked, and gso_segs computed. */
1234 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
1235 skb_shinfo(skb)->gso_segs = 0;
1236
1237 len += vnet_hdr_len;
1238 }
1239
Linus Torvalds1da177e2005-04-16 15:20:36 -07001240 /*
1241 * Now send it
1242 */
1243
1244 err = dev_queue_xmit(skb);
1245 if (err > 0 && (err = net_xmit_errno(err)) != 0)
1246 goto out_unlock;
1247
1248 dev_put(dev);
1249
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001250 return len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001251
1252out_free:
1253 kfree_skb(skb);
1254out_unlock:
1255 if (dev)
1256 dev_put(dev);
1257out:
1258 return err;
1259}
1260
Johann Baudy69e3c752009-05-18 22:11:22 -07001261static int packet_sendmsg(struct kiocb *iocb, struct socket *sock,
1262 struct msghdr *msg, size_t len)
1263{
Johann Baudy69e3c752009-05-18 22:11:22 -07001264 struct sock *sk = sock->sk;
1265 struct packet_sock *po = pkt_sk(sk);
1266 if (po->tx_ring.pg_vec)
1267 return tpacket_snd(po, msg);
1268 else
Johann Baudy69e3c752009-05-18 22:11:22 -07001269 return packet_snd(sock, msg, len);
1270}
1271
Linus Torvalds1da177e2005-04-16 15:20:36 -07001272/*
1273 * Close a PACKET socket. This is fairly simple. We immediately go
1274 * to 'closed' state and remove our protocol entry in the device list.
1275 */
1276
1277static int packet_release(struct socket *sock)
1278{
1279 struct sock *sk = sock->sk;
1280 struct packet_sock *po;
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08001281 struct net *net;
Johann Baudy69e3c752009-05-18 22:11:22 -07001282 struct tpacket_req req;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001283
1284 if (!sk)
1285 return 0;
1286
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09001287 net = sock_net(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001288 po = pkt_sk(sk);
1289
stephen hemminger808f5112010-02-22 07:57:18 +00001290 spin_lock_bh(&net->packet.sklist_lock);
1291 sk_del_node_init_rcu(sk);
Eric Dumazet920de802008-11-24 00:09:29 -08001292 sock_prot_inuse_add(net, sk->sk_prot, -1);
stephen hemminger808f5112010-02-22 07:57:18 +00001293 spin_unlock_bh(&net->packet.sklist_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001294
stephen hemminger808f5112010-02-22 07:57:18 +00001295 spin_lock(&po->bind_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001296 if (po->running) {
1297 /*
stephen hemminger808f5112010-02-22 07:57:18 +00001298 * Remove from protocol table
Linus Torvalds1da177e2005-04-16 15:20:36 -07001299 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001300 po->running = 0;
1301 po->num = 0;
stephen hemminger808f5112010-02-22 07:57:18 +00001302 __dev_remove_pack(&po->prot_hook);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001303 __sock_put(sk);
1304 }
stephen hemminger808f5112010-02-22 07:57:18 +00001305 spin_unlock(&po->bind_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001306
Linus Torvalds1da177e2005-04-16 15:20:36 -07001307 packet_flush_mclist(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001308
Johann Baudy69e3c752009-05-18 22:11:22 -07001309 memset(&req, 0, sizeof(req));
1310
1311 if (po->rx_ring.pg_vec)
1312 packet_set_ring(sk, &req, 1, 0);
1313
1314 if (po->tx_ring.pg_vec)
1315 packet_set_ring(sk, &req, 1, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001316
stephen hemminger808f5112010-02-22 07:57:18 +00001317 synchronize_net();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001318 /*
1319 * Now the socket is dead. No more input will appear.
1320 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001321 sock_orphan(sk);
1322 sock->sk = NULL;
1323
1324 /* Purge queues */
1325
1326 skb_queue_purge(&sk->sk_receive_queue);
Pavel Emelyanov17ab56a2007-11-10 21:38:48 -08001327 sk_refcnt_debug_release(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001328
1329 sock_put(sk);
1330 return 0;
1331}
1332
1333/*
1334 * Attach a packet hook.
1335 */
1336
Al Viro0e11c912006-11-08 00:26:29 -08001337static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 protocol)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001338{
1339 struct packet_sock *po = pkt_sk(sk);
1340 /*
1341 * Detach an existing hook if present.
1342 */
1343
1344 lock_sock(sk);
1345
1346 spin_lock(&po->bind_lock);
1347 if (po->running) {
1348 __sock_put(sk);
1349 po->running = 0;
1350 po->num = 0;
1351 spin_unlock(&po->bind_lock);
1352 dev_remove_pack(&po->prot_hook);
1353 spin_lock(&po->bind_lock);
1354 }
1355
1356 po->num = protocol;
1357 po->prot_hook.type = protocol;
1358 po->prot_hook.dev = dev;
1359
1360 po->ifindex = dev ? dev->ifindex : 0;
1361
1362 if (protocol == 0)
1363 goto out_unlock;
1364
Urs Thuermannbe85d4a2007-11-12 21:05:20 -08001365 if (!dev || (dev->flags & IFF_UP)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001366 dev_add_pack(&po->prot_hook);
1367 sock_hold(sk);
1368 po->running = 1;
Urs Thuermannbe85d4a2007-11-12 21:05:20 -08001369 } else {
1370 sk->sk_err = ENETDOWN;
1371 if (!sock_flag(sk, SOCK_DEAD))
1372 sk->sk_error_report(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001373 }
1374
1375out_unlock:
1376 spin_unlock(&po->bind_lock);
1377 release_sock(sk);
1378 return 0;
1379}
1380
1381/*
1382 * Bind a packet socket to a device
1383 */
1384
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001385static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
1386 int addr_len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001387{
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001388 struct sock *sk = sock->sk;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001389 char name[15];
1390 struct net_device *dev;
1391 int err = -ENODEV;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001392
Linus Torvalds1da177e2005-04-16 15:20:36 -07001393 /*
1394 * Check legality
1395 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001396
Kris Katterjohn8ae55f02006-01-23 16:28:02 -08001397 if (addr_len != sizeof(struct sockaddr))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001398 return -EINVAL;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001399 strlcpy(name, uaddr->sa_data, sizeof(name));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001400
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09001401 dev = dev_get_by_name(sock_net(sk), name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001402 if (dev) {
1403 err = packet_do_bind(sk, dev, pkt_sk(sk)->num);
1404 dev_put(dev);
1405 }
1406 return err;
1407}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001408
1409static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
1410{
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001411 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
1412 struct sock *sk = sock->sk;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001413 struct net_device *dev = NULL;
1414 int err;
1415
1416
1417 /*
1418 * Check legality
1419 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001420
Linus Torvalds1da177e2005-04-16 15:20:36 -07001421 if (addr_len < sizeof(struct sockaddr_ll))
1422 return -EINVAL;
1423 if (sll->sll_family != AF_PACKET)
1424 return -EINVAL;
1425
1426 if (sll->sll_ifindex) {
1427 err = -ENODEV;
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09001428 dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001429 if (dev == NULL)
1430 goto out;
1431 }
1432 err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num);
1433 if (dev)
1434 dev_put(dev);
1435
1436out:
1437 return err;
1438}
1439
1440static struct proto packet_proto = {
1441 .name = "PACKET",
1442 .owner = THIS_MODULE,
1443 .obj_size = sizeof(struct packet_sock),
1444};
1445
1446/*
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001447 * Create a packet of type SOCK_PACKET.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001448 */
1449
Eric Paris3f378b62009-11-05 22:18:14 -08001450static int packet_create(struct net *net, struct socket *sock, int protocol,
1451 int kern)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001452{
1453 struct sock *sk;
1454 struct packet_sock *po;
Al Viro0e11c912006-11-08 00:26:29 -08001455 __be16 proto = (__force __be16)protocol; /* weird, but documented */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001456 int err;
1457
1458 if (!capable(CAP_NET_RAW))
1459 return -EPERM;
David S. Millerbe020972007-05-29 13:16:31 -07001460 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
1461 sock->type != SOCK_PACKET)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001462 return -ESOCKTNOSUPPORT;
1463
1464 sock->state = SS_UNCONNECTED;
1465
1466 err = -ENOBUFS;
Pavel Emelyanov6257ff22007-11-01 00:39:31 -07001467 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001468 if (sk == NULL)
1469 goto out;
1470
1471 sock->ops = &packet_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001472 if (sock->type == SOCK_PACKET)
1473 sock->ops = &packet_ops_spkt;
David S. Millerbe020972007-05-29 13:16:31 -07001474
Linus Torvalds1da177e2005-04-16 15:20:36 -07001475 sock_init_data(sock, sk);
1476
1477 po = pkt_sk(sk);
1478 sk->sk_family = PF_PACKET;
Al Viro0e11c912006-11-08 00:26:29 -08001479 po->num = proto;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001480
1481 sk->sk_destruct = packet_sock_destruct;
Pavel Emelyanov17ab56a2007-11-10 21:38:48 -08001482 sk_refcnt_debug_inc(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001483
1484 /*
1485 * Attach a protocol block
1486 */
1487
1488 spin_lock_init(&po->bind_lock);
Herbert Xu905db442009-01-30 14:12:06 -08001489 mutex_init(&po->pg_vec_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001490 po->prot_hook.func = packet_rcv;
David S. Millerbe020972007-05-29 13:16:31 -07001491
Linus Torvalds1da177e2005-04-16 15:20:36 -07001492 if (sock->type == SOCK_PACKET)
1493 po->prot_hook.func = packet_rcv_spkt;
David S. Millerbe020972007-05-29 13:16:31 -07001494
Linus Torvalds1da177e2005-04-16 15:20:36 -07001495 po->prot_hook.af_packet_priv = sk;
1496
Al Viro0e11c912006-11-08 00:26:29 -08001497 if (proto) {
1498 po->prot_hook.type = proto;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001499 dev_add_pack(&po->prot_hook);
1500 sock_hold(sk);
1501 po->running = 1;
1502 }
1503
stephen hemminger808f5112010-02-22 07:57:18 +00001504 spin_lock_bh(&net->packet.sklist_lock);
1505 sk_add_node_rcu(sk, &net->packet.sklist);
Eric Dumazet36804532008-11-19 14:25:35 -08001506 sock_prot_inuse_add(net, &packet_proto, 1);
stephen hemminger808f5112010-02-22 07:57:18 +00001507 spin_unlock_bh(&net->packet.sklist_lock);
1508
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001509 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001510out:
1511 return err;
1512}
1513
Richard Cochraned85b562010-04-07 22:41:28 +00001514static int packet_recv_error(struct sock *sk, struct msghdr *msg, int len)
1515{
1516 struct sock_exterr_skb *serr;
1517 struct sk_buff *skb, *skb2;
1518 int copied, err;
1519
1520 err = -EAGAIN;
1521 skb = skb_dequeue(&sk->sk_error_queue);
1522 if (skb == NULL)
1523 goto out;
1524
1525 copied = skb->len;
1526 if (copied > len) {
1527 msg->msg_flags |= MSG_TRUNC;
1528 copied = len;
1529 }
1530 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
1531 if (err)
1532 goto out_free_skb;
1533
1534 sock_recv_timestamp(msg, sk, skb);
1535
1536 serr = SKB_EXT_ERR(skb);
1537 put_cmsg(msg, SOL_PACKET, PACKET_TX_TIMESTAMP,
1538 sizeof(serr->ee), &serr->ee);
1539
1540 msg->msg_flags |= MSG_ERRQUEUE;
1541 err = copied;
1542
1543 /* Reset and regenerate socket error */
1544 spin_lock_bh(&sk->sk_error_queue.lock);
1545 sk->sk_err = 0;
1546 if ((skb2 = skb_peek(&sk->sk_error_queue)) != NULL) {
1547 sk->sk_err = SKB_EXT_ERR(skb2)->ee.ee_errno;
1548 spin_unlock_bh(&sk->sk_error_queue.lock);
1549 sk->sk_error_report(sk);
1550 } else
1551 spin_unlock_bh(&sk->sk_error_queue.lock);
1552
1553out_free_skb:
1554 kfree_skb(skb);
1555out:
1556 return err;
1557}
1558
Linus Torvalds1da177e2005-04-16 15:20:36 -07001559/*
1560 * Pull a packet from our receive queue and hand it to the user.
1561 * If necessary we block.
1562 */
1563
1564static int packet_recvmsg(struct kiocb *iocb, struct socket *sock,
1565 struct msghdr *msg, size_t len, int flags)
1566{
1567 struct sock *sk = sock->sk;
1568 struct sk_buff *skb;
1569 int copied, err;
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07001570 struct sockaddr_ll *sll;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08001571 int vnet_hdr_len = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001572
1573 err = -EINVAL;
Richard Cochraned85b562010-04-07 22:41:28 +00001574 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001575 goto out;
1576
1577#if 0
1578 /* What error should we return now? EUNATTACH? */
1579 if (pkt_sk(sk)->ifindex < 0)
1580 return -ENODEV;
1581#endif
1582
Richard Cochraned85b562010-04-07 22:41:28 +00001583 if (flags & MSG_ERRQUEUE) {
1584 err = packet_recv_error(sk, msg, len);
1585 goto out;
1586 }
1587
Linus Torvalds1da177e2005-04-16 15:20:36 -07001588 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001589 * Call the generic datagram receiver. This handles all sorts
1590 * of horrible races and re-entrancy so we can forget about it
1591 * in the protocol layers.
1592 *
1593 * Now it will return ENETDOWN, if device have just gone down,
1594 * but then it will block.
1595 */
1596
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001597 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001598
1599 /*
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001600 * An error occurred so return it. Because skb_recv_datagram()
Linus Torvalds1da177e2005-04-16 15:20:36 -07001601 * handles the blocking we don't see and worry about blocking
1602 * retries.
1603 */
1604
Kris Katterjohn8ae55f02006-01-23 16:28:02 -08001605 if (skb == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001606 goto out;
1607
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08001608 if (pkt_sk(sk)->has_vnet_hdr) {
1609 struct virtio_net_hdr vnet_hdr = { 0 };
1610
1611 err = -EINVAL;
1612 vnet_hdr_len = sizeof(vnet_hdr);
Mariusz Kozlowski1f18b712010-11-08 11:58:45 +00001613 if (len < vnet_hdr_len)
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08001614 goto out_free;
1615
Mariusz Kozlowski1f18b712010-11-08 11:58:45 +00001616 len -= vnet_hdr_len;
1617
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08001618 if (skb_is_gso(skb)) {
1619 struct skb_shared_info *sinfo = skb_shinfo(skb);
1620
1621 /* This is a hint as to how much should be linear. */
1622 vnet_hdr.hdr_len = skb_headlen(skb);
1623 vnet_hdr.gso_size = sinfo->gso_size;
1624 if (sinfo->gso_type & SKB_GSO_TCPV4)
1625 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
1626 else if (sinfo->gso_type & SKB_GSO_TCPV6)
1627 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
1628 else if (sinfo->gso_type & SKB_GSO_UDP)
1629 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
1630 else if (sinfo->gso_type & SKB_GSO_FCOE)
1631 goto out_free;
1632 else
1633 BUG();
1634 if (sinfo->gso_type & SKB_GSO_TCP_ECN)
1635 vnet_hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
1636 } else
1637 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
1638
1639 if (skb->ip_summed == CHECKSUM_PARTIAL) {
1640 vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
1641 vnet_hdr.csum_start = skb->csum_start -
1642 skb_headroom(skb);
1643 vnet_hdr.csum_offset = skb->csum_offset;
1644 } /* else everything is zero */
1645
1646 err = memcpy_toiovec(msg->msg_iov, (void *)&vnet_hdr,
1647 vnet_hdr_len);
1648 if (err < 0)
1649 goto out_free;
1650 }
1651
Linus Torvalds1da177e2005-04-16 15:20:36 -07001652 /*
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07001653 * If the address length field is there to be filled in, we fill
1654 * it in now.
1655 */
1656
Herbert Xuffbc6112007-02-04 23:33:10 -08001657 sll = &PACKET_SKB_CB(skb)->sa.ll;
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07001658 if (sock->type == SOCK_PACKET)
1659 msg->msg_namelen = sizeof(struct sockaddr_pkt);
1660 else
1661 msg->msg_namelen = sll->sll_halen + offsetof(struct sockaddr_ll, sll_addr);
1662
1663 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001664 * You lose any data beyond the buffer you gave. If it worries a
1665 * user program they can ask the device for its MTU anyway.
1666 */
1667
1668 copied = skb->len;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001669 if (copied > len) {
1670 copied = len;
1671 msg->msg_flags |= MSG_TRUNC;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001672 }
1673
1674 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
1675 if (err)
1676 goto out_free;
1677
Neil Horman3b885782009-10-12 13:26:31 -07001678 sock_recv_ts_and_drops(msg, sk, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001679
1680 if (msg->msg_name)
Herbert Xuffbc6112007-02-04 23:33:10 -08001681 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
1682 msg->msg_namelen);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001683
Herbert Xu8dc41942007-02-04 23:31:32 -08001684 if (pkt_sk(sk)->auxdata) {
Herbert Xuffbc6112007-02-04 23:33:10 -08001685 struct tpacket_auxdata aux;
1686
1687 aux.tp_status = TP_STATUS_USER;
1688 if (skb->ip_summed == CHECKSUM_PARTIAL)
1689 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
1690 aux.tp_len = PACKET_SKB_CB(skb)->origlen;
1691 aux.tp_snaplen = skb->len;
1692 aux.tp_mac = 0;
Arnaldo Carvalho de Melobbe735e2007-03-10 22:16:10 -03001693 aux.tp_net = skb_network_offset(skb);
Eric Dumazet05423b22009-10-26 18:40:35 -07001694 aux.tp_vlan_tci = vlan_tx_tag_get(skb);
Herbert Xuffbc6112007-02-04 23:33:10 -08001695
1696 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
Herbert Xu8dc41942007-02-04 23:31:32 -08001697 }
1698
Linus Torvalds1da177e2005-04-16 15:20:36 -07001699 /*
1700 * Free or return the buffer as appropriate. Again this
1701 * hides all the races and re-entrancy issues from us.
1702 */
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08001703 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001704
1705out_free:
1706 skb_free_datagram(sk, skb);
1707out:
1708 return err;
1709}
1710
Linus Torvalds1da177e2005-04-16 15:20:36 -07001711static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
1712 int *uaddr_len, int peer)
1713{
1714 struct net_device *dev;
1715 struct sock *sk = sock->sk;
1716
1717 if (peer)
1718 return -EOPNOTSUPP;
1719
1720 uaddr->sa_family = AF_PACKET;
Eric Dumazet654d1f82009-11-02 10:43:32 +01001721 rcu_read_lock();
1722 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
1723 if (dev)
Vasiliy Kulikov67286642010-11-10 12:09:10 -08001724 strncpy(uaddr->sa_data, dev->name, 14);
Eric Dumazet654d1f82009-11-02 10:43:32 +01001725 else
Linus Torvalds1da177e2005-04-16 15:20:36 -07001726 memset(uaddr->sa_data, 0, 14);
Eric Dumazet654d1f82009-11-02 10:43:32 +01001727 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001728 *uaddr_len = sizeof(*uaddr);
1729
1730 return 0;
1731}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001732
1733static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
1734 int *uaddr_len, int peer)
1735{
1736 struct net_device *dev;
1737 struct sock *sk = sock->sk;
1738 struct packet_sock *po = pkt_sk(sk);
Cyrill Gorcunov13cfa972009-11-08 05:51:19 +00001739 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001740
1741 if (peer)
1742 return -EOPNOTSUPP;
1743
1744 sll->sll_family = AF_PACKET;
1745 sll->sll_ifindex = po->ifindex;
1746 sll->sll_protocol = po->num;
Vasiliy Kulikov67286642010-11-10 12:09:10 -08001747 sll->sll_pkttype = 0;
Eric Dumazet654d1f82009-11-02 10:43:32 +01001748 rcu_read_lock();
1749 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001750 if (dev) {
1751 sll->sll_hatype = dev->type;
1752 sll->sll_halen = dev->addr_len;
1753 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001754 } else {
1755 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
1756 sll->sll_halen = 0;
1757 }
Eric Dumazet654d1f82009-11-02 10:43:32 +01001758 rcu_read_unlock();
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07001759 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001760
1761 return 0;
1762}
1763
Wang Chen2aeb0b82008-07-14 20:49:46 -07001764static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
1765 int what)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001766{
1767 switch (i->type) {
1768 case PACKET_MR_MULTICAST:
Jiri Pirko11625632010-03-02 20:40:01 +00001769 if (i->alen != dev->addr_len)
1770 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001771 if (what > 0)
Jiri Pirko22bedad2010-04-01 21:22:57 +00001772 return dev_mc_add(dev, i->addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001773 else
Jiri Pirko22bedad2010-04-01 21:22:57 +00001774 return dev_mc_del(dev, i->addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001775 break;
1776 case PACKET_MR_PROMISC:
Wang Chen2aeb0b82008-07-14 20:49:46 -07001777 return dev_set_promiscuity(dev, what);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001778 break;
1779 case PACKET_MR_ALLMULTI:
Wang Chen2aeb0b82008-07-14 20:49:46 -07001780 return dev_set_allmulti(dev, what);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001781 break;
Eric W. Biedermand95ed922009-05-19 18:27:17 +00001782 case PACKET_MR_UNICAST:
Jiri Pirko11625632010-03-02 20:40:01 +00001783 if (i->alen != dev->addr_len)
1784 return -EINVAL;
Eric W. Biedermand95ed922009-05-19 18:27:17 +00001785 if (what > 0)
Jiri Pirkoa748ee22010-04-01 21:22:09 +00001786 return dev_uc_add(dev, i->addr);
Eric W. Biedermand95ed922009-05-19 18:27:17 +00001787 else
Jiri Pirkoa748ee22010-04-01 21:22:09 +00001788 return dev_uc_del(dev, i->addr);
Eric W. Biedermand95ed922009-05-19 18:27:17 +00001789 break;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001790 default:
1791 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001792 }
Wang Chen2aeb0b82008-07-14 20:49:46 -07001793 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001794}
1795
1796static void packet_dev_mclist(struct net_device *dev, struct packet_mclist *i, int what)
1797{
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001798 for ( ; i; i = i->next) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001799 if (i->ifindex == dev->ifindex)
1800 packet_dev_mc(dev, i, what);
1801 }
1802}
1803
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07001804static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001805{
1806 struct packet_sock *po = pkt_sk(sk);
1807 struct packet_mclist *ml, *i;
1808 struct net_device *dev;
1809 int err;
1810
1811 rtnl_lock();
1812
1813 err = -ENODEV;
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09001814 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001815 if (!dev)
1816 goto done;
1817
1818 err = -EINVAL;
Jiri Pirko11625632010-03-02 20:40:01 +00001819 if (mreq->mr_alen > dev->addr_len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001820 goto done;
1821
1822 err = -ENOBUFS;
Kris Katterjohn8b3a7002006-01-11 15:56:43 -08001823 i = kmalloc(sizeof(*i), GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001824 if (i == NULL)
1825 goto done;
1826
1827 err = 0;
1828 for (ml = po->mclist; ml; ml = ml->next) {
1829 if (ml->ifindex == mreq->mr_ifindex &&
1830 ml->type == mreq->mr_type &&
1831 ml->alen == mreq->mr_alen &&
1832 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
1833 ml->count++;
1834 /* Free the new element ... */
1835 kfree(i);
1836 goto done;
1837 }
1838 }
1839
1840 i->type = mreq->mr_type;
1841 i->ifindex = mreq->mr_ifindex;
1842 i->alen = mreq->mr_alen;
1843 memcpy(i->addr, mreq->mr_address, i->alen);
1844 i->count = 1;
1845 i->next = po->mclist;
1846 po->mclist = i;
Wang Chen2aeb0b82008-07-14 20:49:46 -07001847 err = packet_dev_mc(dev, i, 1);
1848 if (err) {
1849 po->mclist = i->next;
1850 kfree(i);
1851 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001852
1853done:
1854 rtnl_unlock();
1855 return err;
1856}
1857
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07001858static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001859{
1860 struct packet_mclist *ml, **mlp;
1861
1862 rtnl_lock();
1863
1864 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
1865 if (ml->ifindex == mreq->mr_ifindex &&
1866 ml->type == mreq->mr_type &&
1867 ml->alen == mreq->mr_alen &&
1868 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
1869 if (--ml->count == 0) {
1870 struct net_device *dev;
1871 *mlp = ml->next;
Eric Dumazetad959e72009-10-16 06:38:46 +00001872 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
1873 if (dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001874 packet_dev_mc(dev, ml, -1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001875 kfree(ml);
1876 }
1877 rtnl_unlock();
1878 return 0;
1879 }
1880 }
1881 rtnl_unlock();
1882 return -EADDRNOTAVAIL;
1883}
1884
1885static void packet_flush_mclist(struct sock *sk)
1886{
1887 struct packet_sock *po = pkt_sk(sk);
1888 struct packet_mclist *ml;
1889
1890 if (!po->mclist)
1891 return;
1892
1893 rtnl_lock();
1894 while ((ml = po->mclist) != NULL) {
1895 struct net_device *dev;
1896
1897 po->mclist = ml->next;
Eric Dumazetad959e72009-10-16 06:38:46 +00001898 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
1899 if (dev != NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001900 packet_dev_mc(dev, ml, -1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001901 kfree(ml);
1902 }
1903 rtnl_unlock();
1904}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001905
1906static int
David S. Millerb7058842009-09-30 16:12:20 -07001907packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001908{
1909 struct sock *sk = sock->sk;
Herbert Xu8dc41942007-02-04 23:31:32 -08001910 struct packet_sock *po = pkt_sk(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001911 int ret;
1912
1913 if (level != SOL_PACKET)
1914 return -ENOPROTOOPT;
1915
Johann Baudy69e3c752009-05-18 22:11:22 -07001916 switch (optname) {
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001917 case PACKET_ADD_MEMBERSHIP:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001918 case PACKET_DROP_MEMBERSHIP:
1919 {
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07001920 struct packet_mreq_max mreq;
1921 int len = optlen;
1922 memset(&mreq, 0, sizeof(mreq));
1923 if (len < sizeof(struct packet_mreq))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001924 return -EINVAL;
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07001925 if (len > sizeof(mreq))
1926 len = sizeof(mreq);
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001927 if (copy_from_user(&mreq, optval, len))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001928 return -EFAULT;
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07001929 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
1930 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001931 if (optname == PACKET_ADD_MEMBERSHIP)
1932 ret = packet_mc_add(sk, &mreq);
1933 else
1934 ret = packet_mc_drop(sk, &mreq);
1935 return ret;
1936 }
David S. Millera2efcfa2007-05-29 13:12:50 -07001937
Linus Torvalds1da177e2005-04-16 15:20:36 -07001938 case PACKET_RX_RING:
Johann Baudy69e3c752009-05-18 22:11:22 -07001939 case PACKET_TX_RING:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001940 {
1941 struct tpacket_req req;
1942
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001943 if (optlen < sizeof(req))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001944 return -EINVAL;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08001945 if (pkt_sk(sk)->has_vnet_hdr)
1946 return -EINVAL;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001947 if (copy_from_user(&req, optval, sizeof(req)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001948 return -EFAULT;
Johann Baudy69e3c752009-05-18 22:11:22 -07001949 return packet_set_ring(sk, &req, 0, optname == PACKET_TX_RING);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001950 }
1951 case PACKET_COPY_THRESH:
1952 {
1953 int val;
1954
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001955 if (optlen != sizeof(val))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001956 return -EINVAL;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001957 if (copy_from_user(&val, optval, sizeof(val)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001958 return -EFAULT;
1959
1960 pkt_sk(sk)->copy_thresh = val;
1961 return 0;
1962 }
Patrick McHardybbd6ef82008-07-14 22:50:15 -07001963 case PACKET_VERSION:
1964 {
1965 int val;
1966
1967 if (optlen != sizeof(val))
1968 return -EINVAL;
Johann Baudy69e3c752009-05-18 22:11:22 -07001969 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
Patrick McHardybbd6ef82008-07-14 22:50:15 -07001970 return -EBUSY;
1971 if (copy_from_user(&val, optval, sizeof(val)))
1972 return -EFAULT;
1973 switch (val) {
1974 case TPACKET_V1:
1975 case TPACKET_V2:
1976 po->tp_version = val;
1977 return 0;
1978 default:
1979 return -EINVAL;
1980 }
1981 }
Patrick McHardy89133362008-07-18 18:05:19 -07001982 case PACKET_RESERVE:
1983 {
1984 unsigned int val;
1985
1986 if (optlen != sizeof(val))
1987 return -EINVAL;
Johann Baudy69e3c752009-05-18 22:11:22 -07001988 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
Patrick McHardy89133362008-07-18 18:05:19 -07001989 return -EBUSY;
1990 if (copy_from_user(&val, optval, sizeof(val)))
1991 return -EFAULT;
1992 po->tp_reserve = val;
1993 return 0;
1994 }
Johann Baudy69e3c752009-05-18 22:11:22 -07001995 case PACKET_LOSS:
1996 {
1997 unsigned int val;
1998
1999 if (optlen != sizeof(val))
2000 return -EINVAL;
2001 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
2002 return -EBUSY;
2003 if (copy_from_user(&val, optval, sizeof(val)))
2004 return -EFAULT;
2005 po->tp_loss = !!val;
2006 return 0;
2007 }
Herbert Xu8dc41942007-02-04 23:31:32 -08002008 case PACKET_AUXDATA:
2009 {
2010 int val;
2011
2012 if (optlen < sizeof(val))
2013 return -EINVAL;
2014 if (copy_from_user(&val, optval, sizeof(val)))
2015 return -EFAULT;
2016
2017 po->auxdata = !!val;
2018 return 0;
2019 }
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07002020 case PACKET_ORIGDEV:
2021 {
2022 int val;
2023
2024 if (optlen < sizeof(val))
2025 return -EINVAL;
2026 if (copy_from_user(&val, optval, sizeof(val)))
2027 return -EFAULT;
2028
2029 po->origdev = !!val;
2030 return 0;
2031 }
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002032 case PACKET_VNET_HDR:
2033 {
2034 int val;
2035
2036 if (sock->type != SOCK_RAW)
2037 return -EINVAL;
2038 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
2039 return -EBUSY;
2040 if (optlen < sizeof(val))
2041 return -EINVAL;
2042 if (copy_from_user(&val, optval, sizeof(val)))
2043 return -EFAULT;
2044
2045 po->has_vnet_hdr = !!val;
2046 return 0;
2047 }
Scott McMillan614f60f2010-06-02 05:53:56 -07002048 case PACKET_TIMESTAMP:
2049 {
2050 int val;
2051
2052 if (optlen != sizeof(val))
2053 return -EINVAL;
2054 if (copy_from_user(&val, optval, sizeof(val)))
2055 return -EFAULT;
2056
2057 po->tp_tstamp = val;
2058 return 0;
2059 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002060 default:
2061 return -ENOPROTOOPT;
2062 }
2063}
2064
2065static int packet_getsockopt(struct socket *sock, int level, int optname,
2066 char __user *optval, int __user *optlen)
2067{
2068 int len;
Herbert Xu8dc41942007-02-04 23:31:32 -08002069 int val;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002070 struct sock *sk = sock->sk;
2071 struct packet_sock *po = pkt_sk(sk);
Herbert Xu8dc41942007-02-04 23:31:32 -08002072 void *data;
2073 struct tpacket_stats st;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002074
2075 if (level != SOL_PACKET)
2076 return -ENOPROTOOPT;
2077
Kris Katterjohn8ae55f02006-01-23 16:28:02 -08002078 if (get_user(len, optlen))
2079 return -EFAULT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002080
2081 if (len < 0)
2082 return -EINVAL;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002083
Johann Baudy69e3c752009-05-18 22:11:22 -07002084 switch (optname) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002085 case PACKET_STATISTICS:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002086 if (len > sizeof(struct tpacket_stats))
2087 len = sizeof(struct tpacket_stats);
2088 spin_lock_bh(&sk->sk_receive_queue.lock);
2089 st = po->stats;
2090 memset(&po->stats, 0, sizeof(st));
2091 spin_unlock_bh(&sk->sk_receive_queue.lock);
2092 st.tp_packets += st.tp_drops;
2093
Herbert Xu8dc41942007-02-04 23:31:32 -08002094 data = &st;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002095 break;
Herbert Xu8dc41942007-02-04 23:31:32 -08002096 case PACKET_AUXDATA:
2097 if (len > sizeof(int))
2098 len = sizeof(int);
2099 val = po->auxdata;
2100
2101 data = &val;
2102 break;
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07002103 case PACKET_ORIGDEV:
2104 if (len > sizeof(int))
2105 len = sizeof(int);
2106 val = po->origdev;
2107
2108 data = &val;
2109 break;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002110 case PACKET_VNET_HDR:
2111 if (len > sizeof(int))
2112 len = sizeof(int);
2113 val = po->has_vnet_hdr;
2114
2115 data = &val;
2116 break;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07002117 case PACKET_VERSION:
2118 if (len > sizeof(int))
2119 len = sizeof(int);
2120 val = po->tp_version;
2121 data = &val;
2122 break;
2123 case PACKET_HDRLEN:
2124 if (len > sizeof(int))
2125 len = sizeof(int);
2126 if (copy_from_user(&val, optval, len))
2127 return -EFAULT;
2128 switch (val) {
2129 case TPACKET_V1:
2130 val = sizeof(struct tpacket_hdr);
2131 break;
2132 case TPACKET_V2:
2133 val = sizeof(struct tpacket2_hdr);
2134 break;
2135 default:
2136 return -EINVAL;
2137 }
2138 data = &val;
2139 break;
Patrick McHardy89133362008-07-18 18:05:19 -07002140 case PACKET_RESERVE:
2141 if (len > sizeof(unsigned int))
2142 len = sizeof(unsigned int);
2143 val = po->tp_reserve;
2144 data = &val;
2145 break;
Johann Baudy69e3c752009-05-18 22:11:22 -07002146 case PACKET_LOSS:
2147 if (len > sizeof(unsigned int))
2148 len = sizeof(unsigned int);
2149 val = po->tp_loss;
2150 data = &val;
2151 break;
Scott McMillan614f60f2010-06-02 05:53:56 -07002152 case PACKET_TIMESTAMP:
2153 if (len > sizeof(int))
2154 len = sizeof(int);
2155 val = po->tp_tstamp;
2156 data = &val;
2157 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002158 default:
2159 return -ENOPROTOOPT;
2160 }
2161
Kris Katterjohn8ae55f02006-01-23 16:28:02 -08002162 if (put_user(len, optlen))
2163 return -EFAULT;
Herbert Xu8dc41942007-02-04 23:31:32 -08002164 if (copy_to_user(optval, data, len))
2165 return -EFAULT;
Kris Katterjohn8ae55f02006-01-23 16:28:02 -08002166 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002167}
2168
2169
2170static int packet_notifier(struct notifier_block *this, unsigned long msg, void *data)
2171{
2172 struct sock *sk;
2173 struct hlist_node *node;
Jason Lunzad930652007-02-20 23:19:54 -08002174 struct net_device *dev = data;
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09002175 struct net *net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002176
stephen hemminger808f5112010-02-22 07:57:18 +00002177 rcu_read_lock();
2178 sk_for_each_rcu(sk, node, &net->packet.sklist) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002179 struct packet_sock *po = pkt_sk(sk);
2180
2181 switch (msg) {
2182 case NETDEV_UNREGISTER:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002183 if (po->mclist)
2184 packet_dev_mclist(dev, po->mclist, -1);
David S. Millera2efcfa2007-05-29 13:12:50 -07002185 /* fallthrough */
2186
Linus Torvalds1da177e2005-04-16 15:20:36 -07002187 case NETDEV_DOWN:
2188 if (dev->ifindex == po->ifindex) {
2189 spin_lock(&po->bind_lock);
2190 if (po->running) {
2191 __dev_remove_pack(&po->prot_hook);
2192 __sock_put(sk);
2193 po->running = 0;
2194 sk->sk_err = ENETDOWN;
2195 if (!sock_flag(sk, SOCK_DEAD))
2196 sk->sk_error_report(sk);
2197 }
2198 if (msg == NETDEV_UNREGISTER) {
2199 po->ifindex = -1;
2200 po->prot_hook.dev = NULL;
2201 }
2202 spin_unlock(&po->bind_lock);
2203 }
2204 break;
2205 case NETDEV_UP:
stephen hemminger808f5112010-02-22 07:57:18 +00002206 if (dev->ifindex == po->ifindex) {
2207 spin_lock(&po->bind_lock);
2208 if (po->num && !po->running) {
2209 dev_add_pack(&po->prot_hook);
2210 sock_hold(sk);
2211 po->running = 1;
2212 }
2213 spin_unlock(&po->bind_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002214 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002215 break;
2216 }
2217 }
stephen hemminger808f5112010-02-22 07:57:18 +00002218 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002219 return NOTIFY_DONE;
2220}
2221
2222
2223static int packet_ioctl(struct socket *sock, unsigned int cmd,
2224 unsigned long arg)
2225{
2226 struct sock *sk = sock->sk;
2227
Johann Baudy69e3c752009-05-18 22:11:22 -07002228 switch (cmd) {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002229 case SIOCOUTQ:
2230 {
2231 int amount = sk_wmem_alloc_get(sk);
Eric Dumazet31e6d362009-06-17 19:05:41 -07002232
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002233 return put_user(amount, (int __user *)arg);
2234 }
2235 case SIOCINQ:
2236 {
2237 struct sk_buff *skb;
2238 int amount = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002239
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002240 spin_lock_bh(&sk->sk_receive_queue.lock);
2241 skb = skb_peek(&sk->sk_receive_queue);
2242 if (skb)
2243 amount = skb->len;
2244 spin_unlock_bh(&sk->sk_receive_queue.lock);
2245 return put_user(amount, (int __user *)arg);
2246 }
2247 case SIOCGSTAMP:
2248 return sock_get_timestamp(sk, (struct timeval __user *)arg);
2249 case SIOCGSTAMPNS:
2250 return sock_get_timestampns(sk, (struct timespec __user *)arg);
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002251
Linus Torvalds1da177e2005-04-16 15:20:36 -07002252#ifdef CONFIG_INET
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002253 case SIOCADDRT:
2254 case SIOCDELRT:
2255 case SIOCDARP:
2256 case SIOCGARP:
2257 case SIOCSARP:
2258 case SIOCGIFADDR:
2259 case SIOCSIFADDR:
2260 case SIOCGIFBRDADDR:
2261 case SIOCSIFBRDADDR:
2262 case SIOCGIFNETMASK:
2263 case SIOCSIFNETMASK:
2264 case SIOCGIFDSTADDR:
2265 case SIOCSIFDSTADDR:
2266 case SIOCSIFFLAGS:
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002267 return inet_dgram_ops.ioctl(sock, cmd, arg);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002268#endif
2269
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002270 default:
2271 return -ENOIOCTLCMD;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002272 }
2273 return 0;
2274}
2275
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002276static unsigned int packet_poll(struct file *file, struct socket *sock,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002277 poll_table *wait)
2278{
2279 struct sock *sk = sock->sk;
2280 struct packet_sock *po = pkt_sk(sk);
2281 unsigned int mask = datagram_poll(file, sock, wait);
2282
2283 spin_lock_bh(&sk->sk_receive_queue.lock);
Johann Baudy69e3c752009-05-18 22:11:22 -07002284 if (po->rx_ring.pg_vec) {
2285 if (!packet_previous_frame(po, &po->rx_ring, TP_STATUS_KERNEL))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002286 mask |= POLLIN | POLLRDNORM;
2287 }
2288 spin_unlock_bh(&sk->sk_receive_queue.lock);
Johann Baudy69e3c752009-05-18 22:11:22 -07002289 spin_lock_bh(&sk->sk_write_queue.lock);
2290 if (po->tx_ring.pg_vec) {
2291 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
2292 mask |= POLLOUT | POLLWRNORM;
2293 }
2294 spin_unlock_bh(&sk->sk_write_queue.lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002295 return mask;
2296}
2297
2298
2299/* Dirty? Well, I still did not learn better way to account
2300 * for user mmaps.
2301 */
2302
2303static void packet_mm_open(struct vm_area_struct *vma)
2304{
2305 struct file *file = vma->vm_file;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002306 struct socket *sock = file->private_data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002307 struct sock *sk = sock->sk;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002308
Linus Torvalds1da177e2005-04-16 15:20:36 -07002309 if (sk)
2310 atomic_inc(&pkt_sk(sk)->mapped);
2311}
2312
2313static void packet_mm_close(struct vm_area_struct *vma)
2314{
2315 struct file *file = vma->vm_file;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002316 struct socket *sock = file->private_data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002317 struct sock *sk = sock->sk;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002318
Linus Torvalds1da177e2005-04-16 15:20:36 -07002319 if (sk)
2320 atomic_dec(&pkt_sk(sk)->mapped);
2321}
2322
Alexey Dobriyanf0f37e22009-09-27 22:29:37 +04002323static const struct vm_operations_struct packet_mmap_ops = {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002324 .open = packet_mm_open,
2325 .close = packet_mm_close,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002326};
2327
David S. Miller4ebf0ae2005-12-06 16:38:35 -08002328static void free_pg_vec(char **pg_vec, unsigned int order, unsigned int len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002329{
2330 int i;
2331
David S. Miller4ebf0ae2005-12-06 16:38:35 -08002332 for (i = 0; i < len; i++) {
2333 if (likely(pg_vec[i]))
2334 free_pages((unsigned long) pg_vec[i], order);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002335 }
2336 kfree(pg_vec);
2337}
2338
David S. Miller4ebf0ae2005-12-06 16:38:35 -08002339static inline char *alloc_one_pg_vec_page(unsigned long order)
2340{
Eric Dumazet719bfea2009-04-15 03:39:52 -07002341 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP | __GFP_ZERO | __GFP_NOWARN;
2342
2343 return (char *) __get_free_pages(gfp_flags, order);
David S. Miller4ebf0ae2005-12-06 16:38:35 -08002344}
2345
2346static char **alloc_pg_vec(struct tpacket_req *req, int order)
2347{
2348 unsigned int block_nr = req->tp_block_nr;
2349 char **pg_vec;
2350 int i;
2351
2352 pg_vec = kzalloc(block_nr * sizeof(char *), GFP_KERNEL);
2353 if (unlikely(!pg_vec))
2354 goto out;
2355
2356 for (i = 0; i < block_nr; i++) {
2357 pg_vec[i] = alloc_one_pg_vec_page(order);
2358 if (unlikely(!pg_vec[i]))
2359 goto out_free_pgvec;
2360 }
2361
2362out:
2363 return pg_vec;
2364
2365out_free_pgvec:
2366 free_pg_vec(pg_vec, order, block_nr);
2367 pg_vec = NULL;
2368 goto out;
2369}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002370
Johann Baudy69e3c752009-05-18 22:11:22 -07002371static int packet_set_ring(struct sock *sk, struct tpacket_req *req,
2372 int closing, int tx_ring)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002373{
2374 char **pg_vec = NULL;
2375 struct packet_sock *po = pkt_sk(sk);
Al Viro0e11c912006-11-08 00:26:29 -08002376 int was_running, order = 0;
Johann Baudy69e3c752009-05-18 22:11:22 -07002377 struct packet_ring_buffer *rb;
2378 struct sk_buff_head *rb_queue;
Al Viro0e11c912006-11-08 00:26:29 -08002379 __be16 num;
Johann Baudy69e3c752009-05-18 22:11:22 -07002380 int err;
2381
2382 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
2383 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
2384
2385 err = -EBUSY;
2386 if (!closing) {
2387 if (atomic_read(&po->mapped))
2388 goto out;
2389 if (atomic_read(&rb->pending))
2390 goto out;
2391 }
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002392
Linus Torvalds1da177e2005-04-16 15:20:36 -07002393 if (req->tp_block_nr) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002394 /* Sanity tests and some calculations */
Johann Baudy69e3c752009-05-18 22:11:22 -07002395 err = -EBUSY;
2396 if (unlikely(rb->pg_vec))
2397 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002398
Patrick McHardybbd6ef82008-07-14 22:50:15 -07002399 switch (po->tp_version) {
2400 case TPACKET_V1:
2401 po->tp_hdrlen = TPACKET_HDRLEN;
2402 break;
2403 case TPACKET_V2:
2404 po->tp_hdrlen = TPACKET2_HDRLEN;
2405 break;
2406 }
2407
Johann Baudy69e3c752009-05-18 22:11:22 -07002408 err = -EINVAL;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08002409 if (unlikely((int)req->tp_block_size <= 0))
Johann Baudy69e3c752009-05-18 22:11:22 -07002410 goto out;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08002411 if (unlikely(req->tp_block_size & (PAGE_SIZE - 1)))
Johann Baudy69e3c752009-05-18 22:11:22 -07002412 goto out;
Patrick McHardy89133362008-07-18 18:05:19 -07002413 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
Johann Baudy69e3c752009-05-18 22:11:22 -07002414 po->tp_reserve))
2415 goto out;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08002416 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
Johann Baudy69e3c752009-05-18 22:11:22 -07002417 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002418
Johann Baudy69e3c752009-05-18 22:11:22 -07002419 rb->frames_per_block = req->tp_block_size/req->tp_frame_size;
2420 if (unlikely(rb->frames_per_block <= 0))
2421 goto out;
2422 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
2423 req->tp_frame_nr))
2424 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002425
2426 err = -ENOMEM;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08002427 order = get_order(req->tp_block_size);
2428 pg_vec = alloc_pg_vec(req, order);
2429 if (unlikely(!pg_vec))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002430 goto out;
Johann Baudy69e3c752009-05-18 22:11:22 -07002431 }
2432 /* Done */
2433 else {
2434 err = -EINVAL;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08002435 if (unlikely(req->tp_frame_nr))
Johann Baudy69e3c752009-05-18 22:11:22 -07002436 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002437 }
2438
2439 lock_sock(sk);
2440
2441 /* Detach socket from network */
2442 spin_lock(&po->bind_lock);
2443 was_running = po->running;
2444 num = po->num;
2445 if (was_running) {
2446 __dev_remove_pack(&po->prot_hook);
2447 po->num = 0;
2448 po->running = 0;
2449 __sock_put(sk);
2450 }
2451 spin_unlock(&po->bind_lock);
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002452
Linus Torvalds1da177e2005-04-16 15:20:36 -07002453 synchronize_net();
2454
2455 err = -EBUSY;
Herbert Xu905db442009-01-30 14:12:06 -08002456 mutex_lock(&po->pg_vec_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002457 if (closing || atomic_read(&po->mapped) == 0) {
2458 err = 0;
2459#define XC(a, b) ({ __typeof__ ((a)) __t; __t = (a); (a) = (b); __t; })
Johann Baudy69e3c752009-05-18 22:11:22 -07002460 spin_lock_bh(&rb_queue->lock);
2461 pg_vec = XC(rb->pg_vec, pg_vec);
2462 rb->frame_max = (req->tp_frame_nr - 1);
2463 rb->head = 0;
2464 rb->frame_size = req->tp_frame_size;
2465 spin_unlock_bh(&rb_queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002466
Johann Baudy69e3c752009-05-18 22:11:22 -07002467 order = XC(rb->pg_vec_order, order);
2468 req->tp_block_nr = XC(rb->pg_vec_len, req->tp_block_nr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002469
Johann Baudy69e3c752009-05-18 22:11:22 -07002470 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
2471 po->prot_hook.func = (po->rx_ring.pg_vec) ?
2472 tpacket_rcv : packet_rcv;
2473 skb_queue_purge(rb_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002474#undef XC
2475 if (atomic_read(&po->mapped))
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002476 pr_err("packet_mmap: vma is busy: %d\n",
2477 atomic_read(&po->mapped));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002478 }
Herbert Xu905db442009-01-30 14:12:06 -08002479 mutex_unlock(&po->pg_vec_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002480
2481 spin_lock(&po->bind_lock);
2482 if (was_running && !po->running) {
2483 sock_hold(sk);
2484 po->running = 1;
2485 po->num = num;
2486 dev_add_pack(&po->prot_hook);
2487 }
2488 spin_unlock(&po->bind_lock);
2489
2490 release_sock(sk);
2491
Linus Torvalds1da177e2005-04-16 15:20:36 -07002492 if (pg_vec)
2493 free_pg_vec(pg_vec, order, req->tp_block_nr);
2494out:
2495 return err;
2496}
2497
Johann Baudy69e3c752009-05-18 22:11:22 -07002498static int packet_mmap(struct file *file, struct socket *sock,
2499 struct vm_area_struct *vma)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002500{
2501 struct sock *sk = sock->sk;
2502 struct packet_sock *po = pkt_sk(sk);
Johann Baudy69e3c752009-05-18 22:11:22 -07002503 unsigned long size, expected_size;
2504 struct packet_ring_buffer *rb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002505 unsigned long start;
2506 int err = -EINVAL;
2507 int i;
2508
2509 if (vma->vm_pgoff)
2510 return -EINVAL;
2511
Herbert Xu905db442009-01-30 14:12:06 -08002512 mutex_lock(&po->pg_vec_lock);
Johann Baudy69e3c752009-05-18 22:11:22 -07002513
2514 expected_size = 0;
2515 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
2516 if (rb->pg_vec) {
2517 expected_size += rb->pg_vec_len
2518 * rb->pg_vec_pages
2519 * PAGE_SIZE;
2520 }
2521 }
2522
2523 if (expected_size == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002524 goto out;
Johann Baudy69e3c752009-05-18 22:11:22 -07002525
2526 size = vma->vm_end - vma->vm_start;
2527 if (size != expected_size)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002528 goto out;
2529
Linus Torvalds1da177e2005-04-16 15:20:36 -07002530 start = vma->vm_start;
Johann Baudy69e3c752009-05-18 22:11:22 -07002531 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
2532 if (rb->pg_vec == NULL)
2533 continue;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08002534
Johann Baudy69e3c752009-05-18 22:11:22 -07002535 for (i = 0; i < rb->pg_vec_len; i++) {
2536 struct page *page = virt_to_page(rb->pg_vec[i]);
2537 int pg_num;
2538
2539 for (pg_num = 0; pg_num < rb->pg_vec_pages;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002540 pg_num++, page++) {
Johann Baudy69e3c752009-05-18 22:11:22 -07002541 err = vm_insert_page(vma, start, page);
2542 if (unlikely(err))
2543 goto out;
2544 start += PAGE_SIZE;
2545 }
David S. Miller4ebf0ae2005-12-06 16:38:35 -08002546 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002547 }
Johann Baudy69e3c752009-05-18 22:11:22 -07002548
David S. Miller4ebf0ae2005-12-06 16:38:35 -08002549 atomic_inc(&po->mapped);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002550 vma->vm_ops = &packet_mmap_ops;
2551 err = 0;
2552
2553out:
Herbert Xu905db442009-01-30 14:12:06 -08002554 mutex_unlock(&po->pg_vec_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002555 return err;
2556}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002557
Eric Dumazet90ddc4f2005-12-22 12:49:22 -08002558static const struct proto_ops packet_ops_spkt = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002559 .family = PF_PACKET,
2560 .owner = THIS_MODULE,
2561 .release = packet_release,
2562 .bind = packet_bind_spkt,
2563 .connect = sock_no_connect,
2564 .socketpair = sock_no_socketpair,
2565 .accept = sock_no_accept,
2566 .getname = packet_getname_spkt,
2567 .poll = datagram_poll,
2568 .ioctl = packet_ioctl,
2569 .listen = sock_no_listen,
2570 .shutdown = sock_no_shutdown,
2571 .setsockopt = sock_no_setsockopt,
2572 .getsockopt = sock_no_getsockopt,
2573 .sendmsg = packet_sendmsg_spkt,
2574 .recvmsg = packet_recvmsg,
2575 .mmap = sock_no_mmap,
2576 .sendpage = sock_no_sendpage,
2577};
Linus Torvalds1da177e2005-04-16 15:20:36 -07002578
Eric Dumazet90ddc4f2005-12-22 12:49:22 -08002579static const struct proto_ops packet_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002580 .family = PF_PACKET,
2581 .owner = THIS_MODULE,
2582 .release = packet_release,
2583 .bind = packet_bind,
2584 .connect = sock_no_connect,
2585 .socketpair = sock_no_socketpair,
2586 .accept = sock_no_accept,
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002587 .getname = packet_getname,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002588 .poll = packet_poll,
2589 .ioctl = packet_ioctl,
2590 .listen = sock_no_listen,
2591 .shutdown = sock_no_shutdown,
2592 .setsockopt = packet_setsockopt,
2593 .getsockopt = packet_getsockopt,
2594 .sendmsg = packet_sendmsg,
2595 .recvmsg = packet_recvmsg,
2596 .mmap = packet_mmap,
2597 .sendpage = sock_no_sendpage,
2598};
2599
Stephen Hemmingerec1b4cf2009-10-05 05:58:39 +00002600static const struct net_proto_family packet_family_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002601 .family = PF_PACKET,
2602 .create = packet_create,
2603 .owner = THIS_MODULE,
2604};
2605
2606static struct notifier_block packet_netdev_notifier = {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002607 .notifier_call = packet_notifier,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002608};
2609
2610#ifdef CONFIG_PROC_FS
Linus Torvalds1da177e2005-04-16 15:20:36 -07002611
2612static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
stephen hemminger808f5112010-02-22 07:57:18 +00002613 __acquires(RCU)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002614{
Denis V. Luneve372c412007-11-19 22:31:54 -08002615 struct net *net = seq_file_net(seq);
stephen hemminger808f5112010-02-22 07:57:18 +00002616
2617 rcu_read_lock();
2618 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002619}
2620
2621static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
2622{
Herbert Xu1bf40952007-12-16 14:04:02 -08002623 struct net *net = seq_file_net(seq);
stephen hemminger808f5112010-02-22 07:57:18 +00002624 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002625}
2626
2627static void packet_seq_stop(struct seq_file *seq, void *v)
stephen hemminger808f5112010-02-22 07:57:18 +00002628 __releases(RCU)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002629{
stephen hemminger808f5112010-02-22 07:57:18 +00002630 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002631}
2632
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002633static int packet_seq_show(struct seq_file *seq, void *v)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002634{
2635 if (v == SEQ_START_TOKEN)
2636 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
2637 else {
Li Zefanb7ceabd2010-02-08 23:19:29 +00002638 struct sock *s = sk_entry(v);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002639 const struct packet_sock *po = pkt_sk(s);
2640
2641 seq_printf(seq,
2642 "%p %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
2643 s,
2644 atomic_read(&s->sk_refcnt),
2645 s->sk_type,
2646 ntohs(po->num),
2647 po->ifindex,
2648 po->running,
2649 atomic_read(&s->sk_rmem_alloc),
2650 sock_i_uid(s),
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002651 sock_i_ino(s));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002652 }
2653
2654 return 0;
2655}
2656
Philippe De Muyter56b3d972007-07-10 23:07:31 -07002657static const struct seq_operations packet_seq_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002658 .start = packet_seq_start,
2659 .next = packet_seq_next,
2660 .stop = packet_seq_stop,
2661 .show = packet_seq_show,
2662};
2663
2664static int packet_seq_open(struct inode *inode, struct file *file)
2665{
Denis V. Luneve372c412007-11-19 22:31:54 -08002666 return seq_open_net(inode, file, &packet_seq_ops,
2667 sizeof(struct seq_net_private));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002668}
2669
Arjan van de Venda7071d2007-02-12 00:55:36 -08002670static const struct file_operations packet_seq_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002671 .owner = THIS_MODULE,
2672 .open = packet_seq_open,
2673 .read = seq_read,
2674 .llseek = seq_lseek,
Denis V. Luneve372c412007-11-19 22:31:54 -08002675 .release = seq_release_net,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002676};
2677
2678#endif
2679
Alexey Dobriyan2c8c1e72010-01-17 03:35:32 +00002680static int __net_init packet_net_init(struct net *net)
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08002681{
stephen hemminger808f5112010-02-22 07:57:18 +00002682 spin_lock_init(&net->packet.sklist_lock);
Denis V. Lunev2aaef4e2007-12-11 04:19:54 -08002683 INIT_HLIST_HEAD(&net->packet.sklist);
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08002684
2685 if (!proc_net_fops_create(net, "packet", 0, &packet_seq_fops))
2686 return -ENOMEM;
2687
2688 return 0;
2689}
2690
Alexey Dobriyan2c8c1e72010-01-17 03:35:32 +00002691static void __net_exit packet_net_exit(struct net *net)
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08002692{
2693 proc_net_remove(net, "packet");
2694}
2695
2696static struct pernet_operations packet_net_ops = {
2697 .init = packet_net_init,
2698 .exit = packet_net_exit,
2699};
2700
2701
Linus Torvalds1da177e2005-04-16 15:20:36 -07002702static void __exit packet_exit(void)
2703{
Linus Torvalds1da177e2005-04-16 15:20:36 -07002704 unregister_netdevice_notifier(&packet_netdev_notifier);
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08002705 unregister_pernet_subsys(&packet_net_ops);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002706 sock_unregister(PF_PACKET);
2707 proto_unregister(&packet_proto);
2708}
2709
2710static int __init packet_init(void)
2711{
2712 int rc = proto_register(&packet_proto, 0);
2713
2714 if (rc != 0)
2715 goto out;
2716
2717 sock_register(&packet_family_ops);
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08002718 register_pernet_subsys(&packet_net_ops);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002719 register_netdevice_notifier(&packet_netdev_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002720out:
2721 return rc;
2722}
2723
2724module_init(packet_init);
2725module_exit(packet_exit);
2726MODULE_LICENSE("GPL");
2727MODULE_ALIAS_NETPROTO(PF_PACKET);