blob: 1e0f1c63ad6b33122dad40ce614da0f975720f10 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * Generic socket support routines. Memory allocators, socket lock/release
7 * handler for protocols to use and generic option handler.
8 *
9 *
Jesper Juhl02c30a82005-05-05 16:16:16 -070010 * Authors: Ross Biro
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Florian La Roche, <flla@stud.uni-sb.de>
13 * Alan Cox, <A.Cox@swansea.ac.uk>
14 *
15 * Fixes:
16 * Alan Cox : Numerous verify_area() problems
17 * Alan Cox : Connecting on a connecting socket
18 * now returns an error for tcp.
19 * Alan Cox : sock->protocol is set correctly.
20 * and is not sometimes left as 0.
21 * Alan Cox : connect handles icmp errors on a
22 * connect properly. Unfortunately there
23 * is a restart syscall nasty there. I
24 * can't match BSD without hacking the C
25 * library. Ideas urgently sought!
26 * Alan Cox : Disallow bind() to addresses that are
27 * not ours - especially broadcast ones!!
28 * Alan Cox : Socket 1024 _IS_ ok for users. (fencepost)
29 * Alan Cox : sock_wfree/sock_rfree don't destroy sockets,
30 * instead they leave that for the DESTROY timer.
31 * Alan Cox : Clean up error flag in accept
32 * Alan Cox : TCP ack handling is buggy, the DESTROY timer
33 * was buggy. Put a remove_sock() in the handler
34 * for memory when we hit 0. Also altered the timer
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +090035 * code. The ACK stuff can wait and needs major
Linus Torvalds1da177e2005-04-16 15:20:36 -070036 * TCP layer surgery.
37 * Alan Cox : Fixed TCP ack bug, removed remove sock
38 * and fixed timer/inet_bh race.
39 * Alan Cox : Added zapped flag for TCP
40 * Alan Cox : Move kfree_skb into skbuff.c and tidied up surplus code
41 * Alan Cox : for new sk_buff allocations wmalloc/rmalloc now call alloc_skb
42 * Alan Cox : kfree_s calls now are kfree_skbmem so we can track skb resources
43 * Alan Cox : Supports socket option broadcast now as does udp. Packet and raw need fixing.
44 * Alan Cox : Added RCVBUF,SNDBUF size setting. It suddenly occurred to me how easy it was so...
45 * Rick Sladkey : Relaxed UDP rules for matching packets.
46 * C.E.Hawkins : IFF_PROMISC/SIOCGHWADDR support
47 * Pauline Middelink : identd support
48 * Alan Cox : Fixed connect() taking signals I think.
49 * Alan Cox : SO_LINGER supported
50 * Alan Cox : Error reporting fixes
51 * Anonymous : inet_create tidied up (sk->reuse setting)
52 * Alan Cox : inet sockets don't set sk->type!
53 * Alan Cox : Split socket option code
54 * Alan Cox : Callbacks
55 * Alan Cox : Nagle flag for Charles & Johannes stuff
56 * Alex : Removed restriction on inet fioctl
57 * Alan Cox : Splitting INET from NET core
58 * Alan Cox : Fixed bogus SO_TYPE handling in getsockopt()
59 * Adam Caldwell : Missing return in SO_DONTROUTE/SO_DEBUG code
60 * Alan Cox : Split IP from generic code
61 * Alan Cox : New kfree_skbmem()
62 * Alan Cox : Make SO_DEBUG superuser only.
63 * Alan Cox : Allow anyone to clear SO_DEBUG
64 * (compatibility fix)
65 * Alan Cox : Added optimistic memory grabbing for AF_UNIX throughput.
66 * Alan Cox : Allocator for a socket is settable.
67 * Alan Cox : SO_ERROR includes soft errors.
68 * Alan Cox : Allow NULL arguments on some SO_ opts
69 * Alan Cox : Generic socket allocation to make hooks
70 * easier (suggested by Craig Metz).
71 * Michael Pall : SO_ERROR returns positive errno again
72 * Steve Whitehouse: Added default destructor to free
73 * protocol private data.
74 * Steve Whitehouse: Added various other default routines
75 * common to several socket families.
76 * Chris Evans : Call suser() check last on F_SETOWN
77 * Jay Schulist : Added SO_ATTACH_FILTER and SO_DETACH_FILTER.
78 * Andi Kleen : Add sock_kmalloc()/sock_kfree_s()
79 * Andi Kleen : Fix write_space callback
80 * Chris Evans : Security fixes - signedness again
81 * Arnaldo C. Melo : cleanups, use skb_queue_purge
82 *
83 * To Fix:
84 *
85 *
86 * This program is free software; you can redistribute it and/or
87 * modify it under the terms of the GNU General Public License
88 * as published by the Free Software Foundation; either version
89 * 2 of the License, or (at your option) any later version.
90 */
91
Joe Perchese005d192012-05-16 19:58:40 +000092#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
93
Randy Dunlap4fc268d2006-01-11 12:17:47 -080094#include <linux/capability.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070095#include <linux/errno.h>
Richard Cochrancb820f82013-07-19 19:40:09 +020096#include <linux/errqueue.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070097#include <linux/types.h>
98#include <linux/socket.h>
99#include <linux/in.h>
100#include <linux/kernel.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101#include <linux/module.h>
102#include <linux/proc_fs.h>
103#include <linux/seq_file.h>
104#include <linux/sched.h>
105#include <linux/timer.h>
106#include <linux/string.h>
107#include <linux/sockios.h>
108#include <linux/net.h>
109#include <linux/mm.h>
110#include <linux/slab.h>
111#include <linux/interrupt.h>
112#include <linux/poll.h>
113#include <linux/tcp.h>
114#include <linux/init.h>
Al Viroa1f8e7f72006-10-19 16:08:53 -0400115#include <linux/highmem.h>
Eric W. Biederman3f551f92010-06-13 03:28:59 +0000116#include <linux/user_namespace.h>
Ingo Molnarc5905af2012-02-24 08:31:31 +0100117#include <linux/static_key.h>
David S. Miller3969eb32012-01-09 13:44:23 -0800118#include <linux/memcontrol.h>
David S. Miller8c1ae102012-05-03 02:25:55 -0400119#include <linux/prefetch.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700120
121#include <asm/uaccess.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700122
123#include <linux/netdevice.h>
124#include <net/protocol.h>
125#include <linux/skbuff.h>
Eric W. Biederman457c4cb2007-09-12 12:01:34 +0200126#include <net/net_namespace.h>
Arnaldo Carvalho de Melo2e6599c2005-06-18 22:46:52 -0700127#include <net/request_sock.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700128#include <net/sock.h>
Patrick Ohly20d49472009-02-12 05:03:38 +0000129#include <linux/net_tstamp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700130#include <net/xfrm.h>
131#include <linux/ipsec.h>
Herbert Xuf8451722010-05-24 00:12:34 -0700132#include <net/cls_cgroup.h>
Neil Horman5bc14212011-11-22 05:10:51 +0000133#include <net/netprio_cgroup.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700134
135#include <linux/filter.h>
136
Satoru Moriya3847ce32011-06-17 12:00:03 +0000137#include <trace/events/sock.h>
138
Linus Torvalds1da177e2005-04-16 15:20:36 -0700139#ifdef CONFIG_INET
140#include <net/tcp.h>
141#endif
142
Eliezer Tamir076bb0c2013-07-10 17:13:17 +0300143#include <net/busy_poll.h>
Eliezer Tamir06021292013-06-10 11:39:50 +0300144
Glauber Costa36b77a52011-12-16 00:51:59 +0000145static DEFINE_MUTEX(proto_list_mutex);
Glauber Costad1a4c0b2011-12-11 21:47:04 +0000146static LIST_HEAD(proto_list);
147
Eric W. Biedermana3b299d2014-04-23 14:26:56 -0700148/**
149 * sk_ns_capable - General socket capability test
150 * @sk: Socket to use a capability on or through
151 * @user_ns: The user namespace of the capability to use
152 * @cap: The capability to use
153 *
154 * Test to see if the opener of the socket had when the socket was
155 * created and the current process has the capability @cap in the user
156 * namespace @user_ns.
157 */
158bool sk_ns_capable(const struct sock *sk,
159 struct user_namespace *user_ns, int cap)
160{
161 return file_ns_capable(sk->sk_socket->file, user_ns, cap) &&
162 ns_capable(user_ns, cap);
163}
164EXPORT_SYMBOL(sk_ns_capable);
165
166/**
167 * sk_capable - Socket global capability test
168 * @sk: Socket to use a capability on or through
169 * @cap: The global capbility to use
170 *
171 * Test to see if the opener of the socket had when the socket was
172 * created and the current process has the capability @cap in all user
173 * namespaces.
174 */
175bool sk_capable(const struct sock *sk, int cap)
176{
177 return sk_ns_capable(sk, &init_user_ns, cap);
178}
179EXPORT_SYMBOL(sk_capable);
180
181/**
182 * sk_net_capable - Network namespace socket capability test
183 * @sk: Socket to use a capability on or through
184 * @cap: The capability to use
185 *
186 * Test to see if the opener of the socket had when the socke was created
187 * and the current process has the capability @cap over the network namespace
188 * the socket is a member of.
189 */
190bool sk_net_capable(const struct sock *sk, int cap)
191{
192 return sk_ns_capable(sk, sock_net(sk)->user_ns, cap);
193}
194EXPORT_SYMBOL(sk_net_capable);
195
196
Andrew Mortonc255a452012-07-31 16:43:02 -0700197#ifdef CONFIG_MEMCG_KMEM
Glauber Costa1d62e432012-04-09 19:36:33 -0300198int mem_cgroup_sockets_init(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
Glauber Costad1a4c0b2011-12-11 21:47:04 +0000199{
200 struct proto *proto;
201 int ret = 0;
202
Glauber Costa36b77a52011-12-16 00:51:59 +0000203 mutex_lock(&proto_list_mutex);
Glauber Costad1a4c0b2011-12-11 21:47:04 +0000204 list_for_each_entry(proto, &proto_list, node) {
205 if (proto->init_cgroup) {
Glauber Costa1d62e432012-04-09 19:36:33 -0300206 ret = proto->init_cgroup(memcg, ss);
Glauber Costad1a4c0b2011-12-11 21:47:04 +0000207 if (ret)
208 goto out;
209 }
210 }
211
Glauber Costa36b77a52011-12-16 00:51:59 +0000212 mutex_unlock(&proto_list_mutex);
Glauber Costad1a4c0b2011-12-11 21:47:04 +0000213 return ret;
214out:
215 list_for_each_entry_continue_reverse(proto, &proto_list, node)
216 if (proto->destroy_cgroup)
Glauber Costa1d62e432012-04-09 19:36:33 -0300217 proto->destroy_cgroup(memcg);
Glauber Costa36b77a52011-12-16 00:51:59 +0000218 mutex_unlock(&proto_list_mutex);
Glauber Costad1a4c0b2011-12-11 21:47:04 +0000219 return ret;
220}
221
Glauber Costa1d62e432012-04-09 19:36:33 -0300222void mem_cgroup_sockets_destroy(struct mem_cgroup *memcg)
Glauber Costad1a4c0b2011-12-11 21:47:04 +0000223{
224 struct proto *proto;
225
Glauber Costa36b77a52011-12-16 00:51:59 +0000226 mutex_lock(&proto_list_mutex);
Glauber Costad1a4c0b2011-12-11 21:47:04 +0000227 list_for_each_entry_reverse(proto, &proto_list, node)
228 if (proto->destroy_cgroup)
Glauber Costa1d62e432012-04-09 19:36:33 -0300229 proto->destroy_cgroup(memcg);
Glauber Costa36b77a52011-12-16 00:51:59 +0000230 mutex_unlock(&proto_list_mutex);
Glauber Costad1a4c0b2011-12-11 21:47:04 +0000231}
232#endif
233
Ingo Molnarda21f242006-07-03 00:25:12 -0700234/*
235 * Each address family might have different locking rules, so we have
236 * one slock key per address family:
237 */
Ingo Molnara5b5bb92006-07-03 00:25:35 -0700238static struct lock_class_key af_family_keys[AF_MAX];
239static struct lock_class_key af_family_slock_keys[AF_MAX];
240
stephen hemmingercbda4ea2013-02-22 07:59:10 +0000241#if defined(CONFIG_MEMCG_KMEM)
Ingo Molnarc5905af2012-02-24 08:31:31 +0100242struct static_key memcg_socket_limit_enabled;
Glauber Costae1aab162011-12-11 21:47:03 +0000243EXPORT_SYMBOL(memcg_socket_limit_enabled);
stephen hemmingercbda4ea2013-02-22 07:59:10 +0000244#endif
Glauber Costae1aab162011-12-11 21:47:03 +0000245
Ingo Molnara5b5bb92006-07-03 00:25:35 -0700246/*
247 * Make lock validator output more readable. (we pre-construct these
248 * strings build-time, so that runtime initialization of socket
249 * locks is fast):
250 */
Jan Engelhardt36cbd3d2009-08-05 10:42:58 -0700251static const char *const af_family_key_strings[AF_MAX+1] = {
Ingo Molnara5b5bb92006-07-03 00:25:35 -0700252 "sk_lock-AF_UNSPEC", "sk_lock-AF_UNIX" , "sk_lock-AF_INET" ,
253 "sk_lock-AF_AX25" , "sk_lock-AF_IPX" , "sk_lock-AF_APPLETALK",
254 "sk_lock-AF_NETROM", "sk_lock-AF_BRIDGE" , "sk_lock-AF_ATMPVC" ,
255 "sk_lock-AF_X25" , "sk_lock-AF_INET6" , "sk_lock-AF_ROSE" ,
256 "sk_lock-AF_DECnet", "sk_lock-AF_NETBEUI" , "sk_lock-AF_SECURITY" ,
257 "sk_lock-AF_KEY" , "sk_lock-AF_NETLINK" , "sk_lock-AF_PACKET" ,
258 "sk_lock-AF_ASH" , "sk_lock-AF_ECONET" , "sk_lock-AF_ATMSVC" ,
Andy Grovercbd151b2009-02-26 23:43:19 -0800259 "sk_lock-AF_RDS" , "sk_lock-AF_SNA" , "sk_lock-AF_IRDA" ,
Ingo Molnara5b5bb92006-07-03 00:25:35 -0700260 "sk_lock-AF_PPPOX" , "sk_lock-AF_WANPIPE" , "sk_lock-AF_LLC" ,
Oliver Hartkoppcd05acf2007-12-16 15:59:24 -0800261 "sk_lock-27" , "sk_lock-28" , "sk_lock-AF_CAN" ,
David Howells17926a72007-04-26 15:48:28 -0700262 "sk_lock-AF_TIPC" , "sk_lock-AF_BLUETOOTH", "sk_lock-IUCV" ,
Remi Denis-Courmontbce7b152008-09-22 19:51:15 -0700263 "sk_lock-AF_RXRPC" , "sk_lock-AF_ISDN" , "sk_lock-AF_PHONET" ,
Miloslav Trmač6f107b52010-12-08 14:35:34 +0800264 "sk_lock-AF_IEEE802154", "sk_lock-AF_CAIF" , "sk_lock-AF_ALG" ,
Federico Vaga456db6a2013-05-28 05:02:44 +0000265 "sk_lock-AF_NFC" , "sk_lock-AF_VSOCK" , "sk_lock-AF_MAX"
Ingo Molnara5b5bb92006-07-03 00:25:35 -0700266};
Jan Engelhardt36cbd3d2009-08-05 10:42:58 -0700267static const char *const af_family_slock_key_strings[AF_MAX+1] = {
Ingo Molnara5b5bb92006-07-03 00:25:35 -0700268 "slock-AF_UNSPEC", "slock-AF_UNIX" , "slock-AF_INET" ,
269 "slock-AF_AX25" , "slock-AF_IPX" , "slock-AF_APPLETALK",
270 "slock-AF_NETROM", "slock-AF_BRIDGE" , "slock-AF_ATMPVC" ,
271 "slock-AF_X25" , "slock-AF_INET6" , "slock-AF_ROSE" ,
272 "slock-AF_DECnet", "slock-AF_NETBEUI" , "slock-AF_SECURITY" ,
273 "slock-AF_KEY" , "slock-AF_NETLINK" , "slock-AF_PACKET" ,
274 "slock-AF_ASH" , "slock-AF_ECONET" , "slock-AF_ATMSVC" ,
Andy Grovercbd151b2009-02-26 23:43:19 -0800275 "slock-AF_RDS" , "slock-AF_SNA" , "slock-AF_IRDA" ,
Ingo Molnara5b5bb92006-07-03 00:25:35 -0700276 "slock-AF_PPPOX" , "slock-AF_WANPIPE" , "slock-AF_LLC" ,
Oliver Hartkoppcd05acf2007-12-16 15:59:24 -0800277 "slock-27" , "slock-28" , "slock-AF_CAN" ,
David Howells17926a72007-04-26 15:48:28 -0700278 "slock-AF_TIPC" , "slock-AF_BLUETOOTH", "slock-AF_IUCV" ,
Remi Denis-Courmontbce7b152008-09-22 19:51:15 -0700279 "slock-AF_RXRPC" , "slock-AF_ISDN" , "slock-AF_PHONET" ,
Miloslav Trmač6f107b52010-12-08 14:35:34 +0800280 "slock-AF_IEEE802154", "slock-AF_CAIF" , "slock-AF_ALG" ,
Federico Vaga456db6a2013-05-28 05:02:44 +0000281 "slock-AF_NFC" , "slock-AF_VSOCK" ,"slock-AF_MAX"
Ingo Molnara5b5bb92006-07-03 00:25:35 -0700282};
Jan Engelhardt36cbd3d2009-08-05 10:42:58 -0700283static const char *const af_family_clock_key_strings[AF_MAX+1] = {
Peter Zijlstra443aef0e2007-07-19 01:49:00 -0700284 "clock-AF_UNSPEC", "clock-AF_UNIX" , "clock-AF_INET" ,
285 "clock-AF_AX25" , "clock-AF_IPX" , "clock-AF_APPLETALK",
286 "clock-AF_NETROM", "clock-AF_BRIDGE" , "clock-AF_ATMPVC" ,
287 "clock-AF_X25" , "clock-AF_INET6" , "clock-AF_ROSE" ,
288 "clock-AF_DECnet", "clock-AF_NETBEUI" , "clock-AF_SECURITY" ,
289 "clock-AF_KEY" , "clock-AF_NETLINK" , "clock-AF_PACKET" ,
290 "clock-AF_ASH" , "clock-AF_ECONET" , "clock-AF_ATMSVC" ,
Andy Grovercbd151b2009-02-26 23:43:19 -0800291 "clock-AF_RDS" , "clock-AF_SNA" , "clock-AF_IRDA" ,
Peter Zijlstra443aef0e2007-07-19 01:49:00 -0700292 "clock-AF_PPPOX" , "clock-AF_WANPIPE" , "clock-AF_LLC" ,
Oliver Hartkoppb4942af2008-07-23 14:06:04 -0700293 "clock-27" , "clock-28" , "clock-AF_CAN" ,
David Howellse51f8022007-07-21 19:30:16 -0700294 "clock-AF_TIPC" , "clock-AF_BLUETOOTH", "clock-AF_IUCV" ,
Remi Denis-Courmontbce7b152008-09-22 19:51:15 -0700295 "clock-AF_RXRPC" , "clock-AF_ISDN" , "clock-AF_PHONET" ,
Miloslav Trmač6f107b52010-12-08 14:35:34 +0800296 "clock-AF_IEEE802154", "clock-AF_CAIF" , "clock-AF_ALG" ,
Federico Vaga456db6a2013-05-28 05:02:44 +0000297 "clock-AF_NFC" , "clock-AF_VSOCK" , "clock-AF_MAX"
Peter Zijlstra443aef0e2007-07-19 01:49:00 -0700298};
Ingo Molnarda21f242006-07-03 00:25:12 -0700299
300/*
301 * sk_callback_lock locking rules are per-address-family,
302 * so split the lock classes by using a per-AF key:
303 */
304static struct lock_class_key af_callback_keys[AF_MAX];
305
Linus Torvalds1da177e2005-04-16 15:20:36 -0700306/* Take into consideration the size of the struct sk_buff overhead in the
307 * determination of these values, since that is non-constant across
308 * platforms. This makes socket queueing behavior and performance
309 * not depend upon such differences.
310 */
311#define _SK_MEM_PACKETS 256
Eric Dumazet87fb4b72011-10-13 07:28:54 +0000312#define _SK_MEM_OVERHEAD SKB_TRUESIZE(256)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700313#define SK_WMEM_MAX (_SK_MEM_OVERHEAD * _SK_MEM_PACKETS)
314#define SK_RMEM_MAX (_SK_MEM_OVERHEAD * _SK_MEM_PACKETS)
315
316/* Run time adjustable parameters. */
Brian Haleyab32ea52006-09-22 14:15:41 -0700317__u32 sysctl_wmem_max __read_mostly = SK_WMEM_MAX;
Hans Schillstrom6d8ebc82012-04-30 08:13:50 +0200318EXPORT_SYMBOL(sysctl_wmem_max);
Brian Haleyab32ea52006-09-22 14:15:41 -0700319__u32 sysctl_rmem_max __read_mostly = SK_RMEM_MAX;
Hans Schillstrom6d8ebc82012-04-30 08:13:50 +0200320EXPORT_SYMBOL(sysctl_rmem_max);
Brian Haleyab32ea52006-09-22 14:15:41 -0700321__u32 sysctl_wmem_default __read_mostly = SK_WMEM_MAX;
322__u32 sysctl_rmem_default __read_mostly = SK_RMEM_MAX;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700323
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300324/* Maximal space eaten by iovec or ancillary data plus some space */
Brian Haleyab32ea52006-09-22 14:15:41 -0700325int sysctl_optmem_max __read_mostly = sizeof(unsigned long)*(2*UIO_MAXIOV+512);
Eric Dumazet2a915252009-05-27 11:30:05 +0000326EXPORT_SYMBOL(sysctl_optmem_max);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700327
Mel Gormanc93bdd02012-07-31 16:44:19 -0700328struct static_key memalloc_socks = STATIC_KEY_INIT_FALSE;
329EXPORT_SYMBOL_GPL(memalloc_socks);
330
Mel Gorman7cb02402012-07-31 16:44:16 -0700331/**
332 * sk_set_memalloc - sets %SOCK_MEMALLOC
333 * @sk: socket to set it on
334 *
335 * Set %SOCK_MEMALLOC on a socket for access to emergency reserves.
336 * It's the responsibility of the admin to adjust min_free_kbytes
337 * to meet the requirements
338 */
339void sk_set_memalloc(struct sock *sk)
340{
341 sock_set_flag(sk, SOCK_MEMALLOC);
342 sk->sk_allocation |= __GFP_MEMALLOC;
Mel Gormanc93bdd02012-07-31 16:44:19 -0700343 static_key_slow_inc(&memalloc_socks);
Mel Gorman7cb02402012-07-31 16:44:16 -0700344}
345EXPORT_SYMBOL_GPL(sk_set_memalloc);
346
347void sk_clear_memalloc(struct sock *sk)
348{
349 sock_reset_flag(sk, SOCK_MEMALLOC);
350 sk->sk_allocation &= ~__GFP_MEMALLOC;
Mel Gormanc93bdd02012-07-31 16:44:19 -0700351 static_key_slow_dec(&memalloc_socks);
Mel Gormanc76562b2012-07-31 16:44:41 -0700352
353 /*
354 * SOCK_MEMALLOC is allowed to ignore rmem limits to ensure forward
355 * progress of swapping. However, if SOCK_MEMALLOC is cleared while
356 * it has rmem allocations there is a risk that the user of the
357 * socket cannot make forward progress due to exceeding the rmem
358 * limits. By rights, sk_clear_memalloc() should only be called
359 * on sockets being torn down but warn and reset the accounting if
360 * that assumption breaks.
361 */
362 if (WARN_ON(sk->sk_forward_alloc))
363 sk_mem_reclaim(sk);
Mel Gorman7cb02402012-07-31 16:44:16 -0700364}
365EXPORT_SYMBOL_GPL(sk_clear_memalloc);
366
Mel Gormanb4b9e352012-07-31 16:44:26 -0700367int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb)
368{
369 int ret;
370 unsigned long pflags = current->flags;
371
372 /* these should have been dropped before queueing */
373 BUG_ON(!sock_flag(sk, SOCK_MEMALLOC));
374
375 current->flags |= PF_MEMALLOC;
376 ret = sk->sk_backlog_rcv(sk, skb);
377 tsk_restore_flags(current, pflags, PF_MEMALLOC);
378
379 return ret;
380}
381EXPORT_SYMBOL(__sk_backlog_rcv);
382
Linus Torvalds1da177e2005-04-16 15:20:36 -0700383static int sock_set_timeout(long *timeo_p, char __user *optval, int optlen)
384{
385 struct timeval tv;
386
387 if (optlen < sizeof(tv))
388 return -EINVAL;
389 if (copy_from_user(&tv, optval, sizeof(tv)))
390 return -EFAULT;
Vasily Averinba780732007-05-24 16:58:54 -0700391 if (tv.tv_usec < 0 || tv.tv_usec >= USEC_PER_SEC)
392 return -EDOM;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700393
Vasily Averinba780732007-05-24 16:58:54 -0700394 if (tv.tv_sec < 0) {
Andrew Morton6f11df82007-07-09 13:16:00 -0700395 static int warned __read_mostly;
396
Vasily Averinba780732007-05-24 16:58:54 -0700397 *timeo_p = 0;
Ilpo Järvinen50aab542008-05-02 16:20:10 -0700398 if (warned < 10 && net_ratelimit()) {
Vasily Averinba780732007-05-24 16:58:54 -0700399 warned++;
Joe Perchese005d192012-05-16 19:58:40 +0000400 pr_info("%s: `%s' (pid %d) tries to set negative timeout\n",
401 __func__, current->comm, task_pid_nr(current));
Ilpo Järvinen50aab542008-05-02 16:20:10 -0700402 }
Vasily Averinba780732007-05-24 16:58:54 -0700403 return 0;
404 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700405 *timeo_p = MAX_SCHEDULE_TIMEOUT;
406 if (tv.tv_sec == 0 && tv.tv_usec == 0)
407 return 0;
408 if (tv.tv_sec < (MAX_SCHEDULE_TIMEOUT/HZ - 1))
409 *timeo_p = tv.tv_sec*HZ + (tv.tv_usec+(1000000/HZ-1))/(1000000/HZ);
410 return 0;
411}
412
413static void sock_warn_obsolete_bsdism(const char *name)
414{
415 static int warned;
416 static char warncomm[TASK_COMM_LEN];
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900417 if (strcmp(warncomm, current->comm) && warned < 5) {
418 strcpy(warncomm, current->comm);
Joe Perchese005d192012-05-16 19:58:40 +0000419 pr_warn("process `%s' is using obsolete %s SO_BSDCOMPAT\n",
420 warncomm, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700421 warned++;
422 }
423}
424
Eric Dumazet08e29af2011-11-28 12:04:18 +0000425#define SK_FLAGS_TIMESTAMP ((1UL << SOCK_TIMESTAMP) | (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE))
426
427static void sock_disable_timestamp(struct sock *sk, unsigned long flags)
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900428{
Eric Dumazet08e29af2011-11-28 12:04:18 +0000429 if (sk->sk_flags & flags) {
430 sk->sk_flags &= ~flags;
431 if (!(sk->sk_flags & SK_FLAGS_TIMESTAMP))
Patrick Ohly20d49472009-02-12 05:03:38 +0000432 net_disable_timestamp();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700433 }
434}
435
436
Denis Vlasenkof0088a52006-03-28 01:08:21 -0800437int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
438{
Eric Dumazet766e90372009-10-14 20:40:11 -0700439 int err;
Denis Vlasenkof0088a52006-03-28 01:08:21 -0800440 int skb_len;
Neil Horman3b885782009-10-12 13:26:31 -0700441 unsigned long flags;
442 struct sk_buff_head *list = &sk->sk_receive_queue;
Denis Vlasenkof0088a52006-03-28 01:08:21 -0800443
Eric Dumazet0fd7bac2011-12-21 07:11:44 +0000444 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf) {
Eric Dumazet766e90372009-10-14 20:40:11 -0700445 atomic_inc(&sk->sk_drops);
Satoru Moriya3847ce32011-06-17 12:00:03 +0000446 trace_sock_rcvqueue_full(sk, skb);
Eric Dumazet766e90372009-10-14 20:40:11 -0700447 return -ENOMEM;
Denis Vlasenkof0088a52006-03-28 01:08:21 -0800448 }
449
Dmitry Mishinfda9ef52006-08-31 15:28:39 -0700450 err = sk_filter(sk, skb);
Denis Vlasenkof0088a52006-03-28 01:08:21 -0800451 if (err)
Eric Dumazet766e90372009-10-14 20:40:11 -0700452 return err;
Denis Vlasenkof0088a52006-03-28 01:08:21 -0800453
Mel Gormanc76562b2012-07-31 16:44:41 -0700454 if (!sk_rmem_schedule(sk, skb, skb->truesize)) {
Eric Dumazet766e90372009-10-14 20:40:11 -0700455 atomic_inc(&sk->sk_drops);
456 return -ENOBUFS;
Hideo Aoki3ab224b2007-12-31 00:11:19 -0800457 }
458
Denis Vlasenkof0088a52006-03-28 01:08:21 -0800459 skb->dev = NULL;
460 skb_set_owner_r(skb, sk);
David S. Miller49ad9592008-12-17 22:11:38 -0800461
Denis Vlasenkof0088a52006-03-28 01:08:21 -0800462 /* Cache the SKB length before we tack it onto the receive
463 * queue. Once it is added it no longer belongs to us and
464 * may be freed by other threads of control pulling packets
465 * from the queue.
466 */
467 skb_len = skb->len;
468
Eric Dumazet7fee2262010-05-11 23:19:48 +0000469 /* we escape from rcu protected region, make sure we dont leak
470 * a norefcounted dst
471 */
472 skb_dst_force(skb);
473
Neil Horman3b885782009-10-12 13:26:31 -0700474 spin_lock_irqsave(&list->lock, flags);
475 skb->dropcount = atomic_read(&sk->sk_drops);
476 __skb_queue_tail(list, skb);
477 spin_unlock_irqrestore(&list->lock, flags);
Denis Vlasenkof0088a52006-03-28 01:08:21 -0800478
479 if (!sock_flag(sk, SOCK_DEAD))
David S. Miller676d2362014-04-11 16:15:36 -0400480 sk->sk_data_ready(sk);
Eric Dumazet766e90372009-10-14 20:40:11 -0700481 return 0;
Denis Vlasenkof0088a52006-03-28 01:08:21 -0800482}
483EXPORT_SYMBOL(sock_queue_rcv_skb);
484
Arnaldo Carvalho de Melo58a5a7b2006-11-16 14:06:06 -0200485int sk_receive_skb(struct sock *sk, struct sk_buff *skb, const int nested)
Denis Vlasenkof0088a52006-03-28 01:08:21 -0800486{
487 int rc = NET_RX_SUCCESS;
488
Dmitry Mishinfda9ef52006-08-31 15:28:39 -0700489 if (sk_filter(sk, skb))
Denis Vlasenkof0088a52006-03-28 01:08:21 -0800490 goto discard_and_relse;
491
492 skb->dev = NULL;
493
Sorin Dumitru274f4822014-07-22 21:16:51 +0300494 if (sk_rcvqueues_full(sk, sk->sk_rcvbuf)) {
Eric Dumazetc3774112010-04-27 15:13:20 -0700495 atomic_inc(&sk->sk_drops);
496 goto discard_and_relse;
497 }
Arnaldo Carvalho de Melo58a5a7b2006-11-16 14:06:06 -0200498 if (nested)
499 bh_lock_sock_nested(sk);
500 else
501 bh_lock_sock(sk);
Ingo Molnara5b5bb92006-07-03 00:25:35 -0700502 if (!sock_owned_by_user(sk)) {
503 /*
504 * trylock + unlock semantics:
505 */
506 mutex_acquire(&sk->sk_lock.dep_map, 0, 1, _RET_IP_);
507
Peter Zijlstrac57943a2008-10-07 14:18:42 -0700508 rc = sk_backlog_rcv(sk, skb);
Ingo Molnara5b5bb92006-07-03 00:25:35 -0700509
510 mutex_release(&sk->sk_lock.dep_map, 1, _RET_IP_);
Eric Dumazetf545a382012-04-22 23:34:26 +0000511 } else if (sk_add_backlog(sk, skb, sk->sk_rcvbuf)) {
Zhu Yi8eae9392010-03-04 18:01:40 +0000512 bh_unlock_sock(sk);
513 atomic_inc(&sk->sk_drops);
514 goto discard_and_relse;
515 }
516
Denis Vlasenkof0088a52006-03-28 01:08:21 -0800517 bh_unlock_sock(sk);
518out:
519 sock_put(sk);
520 return rc;
521discard_and_relse:
522 kfree_skb(skb);
523 goto out;
524}
525EXPORT_SYMBOL(sk_receive_skb);
526
527struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie)
528{
Eric Dumazetb6c67122010-04-08 23:03:29 +0000529 struct dst_entry *dst = __sk_dst_get(sk);
Denis Vlasenkof0088a52006-03-28 01:08:21 -0800530
531 if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) {
Krishna Kumare022f0b2009-10-19 23:46:20 +0000532 sk_tx_queue_clear(sk);
Stephen Hemmingera9b3cd72011-08-01 16:19:00 +0000533 RCU_INIT_POINTER(sk->sk_dst_cache, NULL);
Denis Vlasenkof0088a52006-03-28 01:08:21 -0800534 dst_release(dst);
535 return NULL;
536 }
537
538 return dst;
539}
540EXPORT_SYMBOL(__sk_dst_check);
541
542struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie)
543{
544 struct dst_entry *dst = sk_dst_get(sk);
545
546 if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) {
547 sk_dst_reset(sk);
548 dst_release(dst);
549 return NULL;
550 }
551
552 return dst;
553}
554EXPORT_SYMBOL(sk_dst_check);
555
Brian Haleyc91f6df2012-11-26 05:21:08 +0000556static int sock_setbindtodevice(struct sock *sk, char __user *optval,
557 int optlen)
David S. Miller48788092007-09-14 16:41:03 -0700558{
559 int ret = -ENOPROTOOPT;
560#ifdef CONFIG_NETDEVICES
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +0900561 struct net *net = sock_net(sk);
David S. Miller48788092007-09-14 16:41:03 -0700562 char devname[IFNAMSIZ];
563 int index;
564
565 /* Sorry... */
566 ret = -EPERM;
Eric W. Biederman5e1fccc2012-11-16 03:03:04 +0000567 if (!ns_capable(net->user_ns, CAP_NET_RAW))
David S. Miller48788092007-09-14 16:41:03 -0700568 goto out;
569
570 ret = -EINVAL;
571 if (optlen < 0)
572 goto out;
573
574 /* Bind this socket to a particular device like "eth0",
575 * as specified in the passed interface name. If the
576 * name is "" or the option length is zero the socket
577 * is not bound.
578 */
579 if (optlen > IFNAMSIZ - 1)
580 optlen = IFNAMSIZ - 1;
581 memset(devname, 0, sizeof(devname));
582
583 ret = -EFAULT;
584 if (copy_from_user(devname, optval, optlen))
585 goto out;
586
David S. Miller000ba2e2009-11-05 22:37:11 -0800587 index = 0;
588 if (devname[0] != '\0') {
Eric Dumazetbf8e56b2009-11-05 21:03:39 -0800589 struct net_device *dev;
David S. Miller48788092007-09-14 16:41:03 -0700590
Eric Dumazetbf8e56b2009-11-05 21:03:39 -0800591 rcu_read_lock();
592 dev = dev_get_by_name_rcu(net, devname);
593 if (dev)
594 index = dev->ifindex;
595 rcu_read_unlock();
David S. Miller48788092007-09-14 16:41:03 -0700596 ret = -ENODEV;
597 if (!dev)
598 goto out;
David S. Miller48788092007-09-14 16:41:03 -0700599 }
600
601 lock_sock(sk);
602 sk->sk_bound_dev_if = index;
603 sk_dst_reset(sk);
604 release_sock(sk);
605
606 ret = 0;
607
608out:
609#endif
610
611 return ret;
612}
613
Brian Haleyc91f6df2012-11-26 05:21:08 +0000614static int sock_getbindtodevice(struct sock *sk, char __user *optval,
615 int __user *optlen, int len)
616{
617 int ret = -ENOPROTOOPT;
618#ifdef CONFIG_NETDEVICES
619 struct net *net = sock_net(sk);
Brian Haleyc91f6df2012-11-26 05:21:08 +0000620 char devname[IFNAMSIZ];
Brian Haleyc91f6df2012-11-26 05:21:08 +0000621
622 if (sk->sk_bound_dev_if == 0) {
623 len = 0;
624 goto zero;
625 }
626
627 ret = -EINVAL;
628 if (len < IFNAMSIZ)
629 goto out;
630
Nicolas Schichan5dbe7c12013-06-26 17:23:42 +0200631 ret = netdev_get_name(net, devname, sk->sk_bound_dev_if);
632 if (ret)
Brian Haleyc91f6df2012-11-26 05:21:08 +0000633 goto out;
Brian Haleyc91f6df2012-11-26 05:21:08 +0000634
635 len = strlen(devname) + 1;
636
637 ret = -EFAULT;
638 if (copy_to_user(optval, devname, len))
639 goto out;
640
641zero:
642 ret = -EFAULT;
643 if (put_user(len, optlen))
644 goto out;
645
646 ret = 0;
647
648out:
649#endif
650
651 return ret;
652}
653
Pavel Emelyanovc0ef8772007-11-15 03:03:19 -0800654static inline void sock_valbool_flag(struct sock *sk, int bit, int valbool)
655{
656 if (valbool)
657 sock_set_flag(sk, bit);
658 else
659 sock_reset_flag(sk, bit);
660}
661
Linus Torvalds1da177e2005-04-16 15:20:36 -0700662/*
663 * This is meant for all protocols to use and covers goings on
664 * at the socket level. Everything here is generic.
665 */
666
667int sock_setsockopt(struct socket *sock, int level, int optname,
David S. Millerb7058842009-09-30 16:12:20 -0700668 char __user *optval, unsigned int optlen)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700669{
Eric Dumazet2a915252009-05-27 11:30:05 +0000670 struct sock *sk = sock->sk;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700671 int val;
672 int valbool;
673 struct linger ling;
674 int ret = 0;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900675
Linus Torvalds1da177e2005-04-16 15:20:36 -0700676 /*
677 * Options without arguments
678 */
679
David S. Miller48788092007-09-14 16:41:03 -0700680 if (optname == SO_BINDTODEVICE)
Brian Haleyc91f6df2012-11-26 05:21:08 +0000681 return sock_setbindtodevice(sk, optval, optlen);
David S. Miller48788092007-09-14 16:41:03 -0700682
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700683 if (optlen < sizeof(int))
684 return -EINVAL;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900685
Linus Torvalds1da177e2005-04-16 15:20:36 -0700686 if (get_user(val, (int __user *)optval))
687 return -EFAULT;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900688
Eric Dumazet2a915252009-05-27 11:30:05 +0000689 valbool = val ? 1 : 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700690
691 lock_sock(sk);
692
Eric Dumazet2a915252009-05-27 11:30:05 +0000693 switch (optname) {
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700694 case SO_DEBUG:
Eric Dumazet2a915252009-05-27 11:30:05 +0000695 if (val && !capable(CAP_NET_ADMIN))
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700696 ret = -EACCES;
Eric Dumazet2a915252009-05-27 11:30:05 +0000697 else
Pavel Emelyanovc0ef8772007-11-15 03:03:19 -0800698 sock_valbool_flag(sk, SOCK_DBG, valbool);
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700699 break;
700 case SO_REUSEADDR:
Pavel Emelyanov4a17fd52012-04-19 03:39:36 +0000701 sk->sk_reuse = (valbool ? SK_CAN_REUSE : SK_NO_REUSE);
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700702 break;
Tom Herbert055dc212013-01-22 09:49:50 +0000703 case SO_REUSEPORT:
704 sk->sk_reuseport = valbool;
705 break;
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700706 case SO_TYPE:
Jan Engelhardt49c794e2009-08-04 07:28:28 +0000707 case SO_PROTOCOL:
Jan Engelhardt0d6038e2009-08-04 07:28:29 +0000708 case SO_DOMAIN:
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700709 case SO_ERROR:
710 ret = -ENOPROTOOPT;
711 break;
712 case SO_DONTROUTE:
Pavel Emelyanovc0ef8772007-11-15 03:03:19 -0800713 sock_valbool_flag(sk, SOCK_LOCALROUTE, valbool);
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700714 break;
715 case SO_BROADCAST:
716 sock_valbool_flag(sk, SOCK_BROADCAST, valbool);
717 break;
718 case SO_SNDBUF:
719 /* Don't error on this BSD doesn't and if you think
Eric Dumazet82981932012-04-26 20:07:59 +0000720 * about it this is right. Otherwise apps have to
721 * play 'guess the biggest size' games. RCVBUF/SNDBUF
722 * are treated in BSD as hints
723 */
724 val = min_t(u32, val, sysctl_wmem_max);
Patrick McHardyb0573de2005-08-09 19:30:51 -0700725set_sndbuf:
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700726 sk->sk_userlocks |= SOCK_SNDBUF_LOCK;
Eric Dumazet82981932012-04-26 20:07:59 +0000727 sk->sk_sndbuf = max_t(u32, val * 2, SOCK_MIN_SNDBUF);
728 /* Wake up sending tasks if we upped the value. */
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700729 sk->sk_write_space(sk);
730 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700731
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700732 case SO_SNDBUFFORCE:
733 if (!capable(CAP_NET_ADMIN)) {
734 ret = -EPERM;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700735 break;
736 }
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700737 goto set_sndbuf;
738
739 case SO_RCVBUF:
740 /* Don't error on this BSD doesn't and if you think
Eric Dumazet82981932012-04-26 20:07:59 +0000741 * about it this is right. Otherwise apps have to
742 * play 'guess the biggest size' games. RCVBUF/SNDBUF
743 * are treated in BSD as hints
744 */
745 val = min_t(u32, val, sysctl_rmem_max);
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700746set_rcvbuf:
747 sk->sk_userlocks |= SOCK_RCVBUF_LOCK;
748 /*
749 * We double it on the way in to account for
750 * "struct sk_buff" etc. overhead. Applications
751 * assume that the SO_RCVBUF setting they make will
752 * allow that much actual data to be received on that
753 * socket.
754 *
755 * Applications are unaware that "struct sk_buff" and
756 * other overheads allocate from the receive buffer
757 * during socket buffer allocation.
758 *
759 * And after considering the possible alternatives,
760 * returning the value we actually used in getsockopt
761 * is the most desirable behavior.
762 */
Eric Dumazet82981932012-04-26 20:07:59 +0000763 sk->sk_rcvbuf = max_t(u32, val * 2, SOCK_MIN_RCVBUF);
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700764 break;
765
766 case SO_RCVBUFFORCE:
767 if (!capable(CAP_NET_ADMIN)) {
768 ret = -EPERM;
769 break;
770 }
771 goto set_rcvbuf;
772
773 case SO_KEEPALIVE:
774#ifdef CONFIG_INET
Eric Dumazet3e109862012-09-24 07:00:11 +0000775 if (sk->sk_protocol == IPPROTO_TCP &&
776 sk->sk_type == SOCK_STREAM)
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700777 tcp_set_keepalive(sk, valbool);
778#endif
779 sock_valbool_flag(sk, SOCK_KEEPOPEN, valbool);
780 break;
781
782 case SO_OOBINLINE:
783 sock_valbool_flag(sk, SOCK_URGINLINE, valbool);
784 break;
785
786 case SO_NO_CHECK:
Tom Herbert28448b82014-05-23 08:47:19 -0700787 sk->sk_no_check_tx = valbool;
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700788 break;
789
790 case SO_PRIORITY:
Eric W. Biederman5e1fccc2012-11-16 03:03:04 +0000791 if ((val >= 0 && val <= 6) ||
792 ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700793 sk->sk_priority = val;
794 else
795 ret = -EPERM;
796 break;
797
798 case SO_LINGER:
799 if (optlen < sizeof(ling)) {
800 ret = -EINVAL; /* 1003.1g */
801 break;
802 }
Eric Dumazet2a915252009-05-27 11:30:05 +0000803 if (copy_from_user(&ling, optval, sizeof(ling))) {
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700804 ret = -EFAULT;
805 break;
806 }
807 if (!ling.l_onoff)
808 sock_reset_flag(sk, SOCK_LINGER);
809 else {
810#if (BITS_PER_LONG == 32)
811 if ((unsigned int)ling.l_linger >= MAX_SCHEDULE_TIMEOUT/HZ)
812 sk->sk_lingertime = MAX_SCHEDULE_TIMEOUT;
813 else
814#endif
815 sk->sk_lingertime = (unsigned int)ling.l_linger * HZ;
816 sock_set_flag(sk, SOCK_LINGER);
817 }
818 break;
819
820 case SO_BSDCOMPAT:
821 sock_warn_obsolete_bsdism("setsockopt");
822 break;
823
824 case SO_PASSCRED:
825 if (valbool)
826 set_bit(SOCK_PASSCRED, &sock->flags);
827 else
828 clear_bit(SOCK_PASSCRED, &sock->flags);
829 break;
830
831 case SO_TIMESTAMP:
Eric Dumazet92f37fd2007-03-25 22:14:49 -0700832 case SO_TIMESTAMPNS:
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700833 if (valbool) {
Eric Dumazet92f37fd2007-03-25 22:14:49 -0700834 if (optname == SO_TIMESTAMP)
835 sock_reset_flag(sk, SOCK_RCVTSTAMPNS);
836 else
837 sock_set_flag(sk, SOCK_RCVTSTAMPNS);
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700838 sock_set_flag(sk, SOCK_RCVTSTAMP);
Patrick Ohly20d49472009-02-12 05:03:38 +0000839 sock_enable_timestamp(sk, SOCK_TIMESTAMP);
Eric Dumazet92f37fd2007-03-25 22:14:49 -0700840 } else {
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700841 sock_reset_flag(sk, SOCK_RCVTSTAMP);
Eric Dumazet92f37fd2007-03-25 22:14:49 -0700842 sock_reset_flag(sk, SOCK_RCVTSTAMPNS);
843 }
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700844 break;
845
Patrick Ohly20d49472009-02-12 05:03:38 +0000846 case SO_TIMESTAMPING:
847 if (val & ~SOF_TIMESTAMPING_MASK) {
Rémi Denis-Courmontf249fb72009-07-20 00:47:04 +0000848 ret = -EINVAL;
Patrick Ohly20d49472009-02-12 05:03:38 +0000849 break;
850 }
Willem de Bruijn09c2d252014-08-04 22:11:47 -0400851 if (val & SOF_TIMESTAMPING_OPT_ID &&
852 !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID))
853 sk->sk_tskey = 0;
Willem de Bruijnb9f40e22014-08-04 22:11:46 -0400854 sk->sk_tsflags = val;
Patrick Ohly20d49472009-02-12 05:03:38 +0000855 if (val & SOF_TIMESTAMPING_RX_SOFTWARE)
856 sock_enable_timestamp(sk,
857 SOCK_TIMESTAMPING_RX_SOFTWARE);
858 else
859 sock_disable_timestamp(sk,
Eric Dumazet08e29af2011-11-28 12:04:18 +0000860 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE));
Patrick Ohly20d49472009-02-12 05:03:38 +0000861 break;
862
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700863 case SO_RCVLOWAT:
864 if (val < 0)
865 val = INT_MAX;
866 sk->sk_rcvlowat = val ? : 1;
867 break;
868
869 case SO_RCVTIMEO:
870 ret = sock_set_timeout(&sk->sk_rcvtimeo, optval, optlen);
871 break;
872
873 case SO_SNDTIMEO:
874 ret = sock_set_timeout(&sk->sk_sndtimeo, optval, optlen);
875 break;
876
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700877 case SO_ATTACH_FILTER:
878 ret = -EINVAL;
879 if (optlen == sizeof(struct sock_fprog)) {
880 struct sock_fprog fprog;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700881
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700882 ret = -EFAULT;
883 if (copy_from_user(&fprog, optval, sizeof(fprog)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700884 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700885
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700886 ret = sk_attach_filter(&fprog, sk);
887 }
888 break;
889
890 case SO_DETACH_FILTER:
Pavel Emelyanov55b33322007-10-17 21:21:26 -0700891 ret = sk_detach_filter(sk);
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700892 break;
893
Vincent Bernatd59577b2013-01-16 22:55:49 +0100894 case SO_LOCK_FILTER:
895 if (sock_flag(sk, SOCK_FILTER_LOCKED) && !valbool)
896 ret = -EPERM;
897 else
898 sock_valbool_flag(sk, SOCK_FILTER_LOCKED, valbool);
899 break;
900
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700901 case SO_PASSSEC:
902 if (valbool)
903 set_bit(SOCK_PASSSEC, &sock->flags);
904 else
905 clear_bit(SOCK_PASSSEC, &sock->flags);
906 break;
Laszlo Attila Toth4a19ec52008-01-30 19:08:16 -0800907 case SO_MARK:
Eric W. Biederman5e1fccc2012-11-16 03:03:04 +0000908 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
Laszlo Attila Toth4a19ec52008-01-30 19:08:16 -0800909 ret = -EPERM;
Eric Dumazet2a915252009-05-27 11:30:05 +0000910 else
Laszlo Attila Toth4a19ec52008-01-30 19:08:16 -0800911 sk->sk_mark = val;
Laszlo Attila Toth4a19ec52008-01-30 19:08:16 -0800912 break;
Catherine Zhang877ce7c2006-06-29 12:27:47 -0700913
Linus Torvalds1da177e2005-04-16 15:20:36 -0700914 /* We implement the SO_SNDLOWAT etc to
915 not be settable (1003.1g 5.3) */
Neil Horman3b885782009-10-12 13:26:31 -0700916 case SO_RXQ_OVFL:
Johannes Berg8083f0f2011-10-07 03:30:20 +0000917 sock_valbool_flag(sk, SOCK_RXQ_OVFL, valbool);
Neil Horman3b885782009-10-12 13:26:31 -0700918 break;
Johannes Berg6e3e9392011-11-09 10:15:42 +0100919
920 case SO_WIFI_STATUS:
921 sock_valbool_flag(sk, SOCK_WIFI_STATUS, valbool);
922 break;
923
Pavel Emelyanovef64a542012-02-21 07:31:34 +0000924 case SO_PEEK_OFF:
925 if (sock->ops->set_peek_off)
Sasha Levin12663bf2013-12-07 17:26:27 -0500926 ret = sock->ops->set_peek_off(sk, val);
Pavel Emelyanovef64a542012-02-21 07:31:34 +0000927 else
928 ret = -EOPNOTSUPP;
929 break;
Ben Greear3bdc0eb2012-02-11 15:39:30 +0000930
931 case SO_NOFCS:
932 sock_valbool_flag(sk, SOCK_NOFCS, valbool);
933 break;
934
Keller, Jacob E7d4c04f2013-03-28 11:19:25 +0000935 case SO_SELECT_ERR_QUEUE:
936 sock_valbool_flag(sk, SOCK_SELECT_ERR_QUEUE, valbool);
937 break;
938
Cong Wange0d10952013-08-01 11:10:25 +0800939#ifdef CONFIG_NET_RX_BUSY_POLL
Eliezer Tamir64b0dc52013-07-10 17:13:36 +0300940 case SO_BUSY_POLL:
Eliezer Tamirdafcc432013-06-14 16:33:57 +0300941 /* allow unprivileged users to decrease the value */
942 if ((val > sk->sk_ll_usec) && !capable(CAP_NET_ADMIN))
943 ret = -EPERM;
944 else {
945 if (val < 0)
946 ret = -EINVAL;
947 else
948 sk->sk_ll_usec = val;
949 }
950 break;
951#endif
Eric Dumazet62748f32013-09-24 08:20:52 -0700952
953 case SO_MAX_PACING_RATE:
954 sk->sk_max_pacing_rate = val;
955 sk->sk_pacing_rate = min(sk->sk_pacing_rate,
956 sk->sk_max_pacing_rate);
957 break;
958
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700959 default:
960 ret = -ENOPROTOOPT;
961 break;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900962 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700963 release_sock(sk);
964 return ret;
965}
Eric Dumazet2a915252009-05-27 11:30:05 +0000966EXPORT_SYMBOL(sock_setsockopt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700967
968
stephen hemminger8f098982014-01-03 09:17:14 -0800969static void cred_to_ucred(struct pid *pid, const struct cred *cred,
970 struct ucred *ucred)
Eric W. Biederman3f551f92010-06-13 03:28:59 +0000971{
972 ucred->pid = pid_vnr(pid);
973 ucred->uid = ucred->gid = -1;
974 if (cred) {
975 struct user_namespace *current_ns = current_user_ns();
976
Eric W. Biedermanb2e4f542012-05-23 16:39:45 -0600977 ucred->uid = from_kuid_munged(current_ns, cred->euid);
978 ucred->gid = from_kgid_munged(current_ns, cred->egid);
Eric W. Biederman3f551f92010-06-13 03:28:59 +0000979 }
980}
981
Linus Torvalds1da177e2005-04-16 15:20:36 -0700982int sock_getsockopt(struct socket *sock, int level, int optname,
983 char __user *optval, int __user *optlen)
984{
985 struct sock *sk = sock->sk;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900986
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700987 union {
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900988 int val;
989 struct linger ling;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700990 struct timeval tm;
991 } v;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900992
H Hartley Sweeten4d0392b2010-01-15 01:08:58 -0800993 int lv = sizeof(int);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700994 int len;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900995
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700996 if (get_user(len, optlen))
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900997 return -EFAULT;
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700998 if (len < 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700999 return -EINVAL;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001000
Eugene Teo50fee1d2009-02-23 15:38:41 -08001001 memset(&v, 0, sizeof(v));
Clément Lecignedf0bca02009-02-12 16:59:09 -08001002
Eric Dumazet2a915252009-05-27 11:30:05 +00001003 switch (optname) {
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001004 case SO_DEBUG:
1005 v.val = sock_flag(sk, SOCK_DBG);
1006 break;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001007
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001008 case SO_DONTROUTE:
1009 v.val = sock_flag(sk, SOCK_LOCALROUTE);
1010 break;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001011
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001012 case SO_BROADCAST:
Eric Dumazet1b23a5d2012-05-16 05:57:07 +00001013 v.val = sock_flag(sk, SOCK_BROADCAST);
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001014 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001015
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001016 case SO_SNDBUF:
1017 v.val = sk->sk_sndbuf;
1018 break;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001019
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001020 case SO_RCVBUF:
1021 v.val = sk->sk_rcvbuf;
1022 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001023
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001024 case SO_REUSEADDR:
1025 v.val = sk->sk_reuse;
1026 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001027
Tom Herbert055dc212013-01-22 09:49:50 +00001028 case SO_REUSEPORT:
1029 v.val = sk->sk_reuseport;
1030 break;
1031
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001032 case SO_KEEPALIVE:
Eric Dumazet1b23a5d2012-05-16 05:57:07 +00001033 v.val = sock_flag(sk, SOCK_KEEPOPEN);
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001034 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001035
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001036 case SO_TYPE:
1037 v.val = sk->sk_type;
1038 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001039
Jan Engelhardt49c794e2009-08-04 07:28:28 +00001040 case SO_PROTOCOL:
1041 v.val = sk->sk_protocol;
1042 break;
1043
Jan Engelhardt0d6038e2009-08-04 07:28:29 +00001044 case SO_DOMAIN:
1045 v.val = sk->sk_family;
1046 break;
1047
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001048 case SO_ERROR:
1049 v.val = -sock_error(sk);
Eric Dumazet2a915252009-05-27 11:30:05 +00001050 if (v.val == 0)
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001051 v.val = xchg(&sk->sk_err_soft, 0);
1052 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001053
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001054 case SO_OOBINLINE:
Eric Dumazet1b23a5d2012-05-16 05:57:07 +00001055 v.val = sock_flag(sk, SOCK_URGINLINE);
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001056 break;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001057
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001058 case SO_NO_CHECK:
Tom Herbert28448b82014-05-23 08:47:19 -07001059 v.val = sk->sk_no_check_tx;
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001060 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001061
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001062 case SO_PRIORITY:
1063 v.val = sk->sk_priority;
1064 break;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001065
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001066 case SO_LINGER:
1067 lv = sizeof(v.ling);
Eric Dumazet1b23a5d2012-05-16 05:57:07 +00001068 v.ling.l_onoff = sock_flag(sk, SOCK_LINGER);
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001069 v.ling.l_linger = sk->sk_lingertime / HZ;
1070 break;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001071
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001072 case SO_BSDCOMPAT:
1073 sock_warn_obsolete_bsdism("getsockopt");
1074 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001075
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001076 case SO_TIMESTAMP:
Eric Dumazet92f37fd2007-03-25 22:14:49 -07001077 v.val = sock_flag(sk, SOCK_RCVTSTAMP) &&
1078 !sock_flag(sk, SOCK_RCVTSTAMPNS);
1079 break;
1080
1081 case SO_TIMESTAMPNS:
1082 v.val = sock_flag(sk, SOCK_RCVTSTAMPNS);
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001083 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001084
Patrick Ohly20d49472009-02-12 05:03:38 +00001085 case SO_TIMESTAMPING:
Willem de Bruijnb9f40e22014-08-04 22:11:46 -04001086 v.val = sk->sk_tsflags;
Patrick Ohly20d49472009-02-12 05:03:38 +00001087 break;
1088
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001089 case SO_RCVTIMEO:
Eric Dumazet2a915252009-05-27 11:30:05 +00001090 lv = sizeof(struct timeval);
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001091 if (sk->sk_rcvtimeo == MAX_SCHEDULE_TIMEOUT) {
1092 v.tm.tv_sec = 0;
1093 v.tm.tv_usec = 0;
1094 } else {
1095 v.tm.tv_sec = sk->sk_rcvtimeo / HZ;
1096 v.tm.tv_usec = ((sk->sk_rcvtimeo % HZ) * 1000000) / HZ;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001097 }
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001098 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001099
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001100 case SO_SNDTIMEO:
Eric Dumazet2a915252009-05-27 11:30:05 +00001101 lv = sizeof(struct timeval);
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001102 if (sk->sk_sndtimeo == MAX_SCHEDULE_TIMEOUT) {
1103 v.tm.tv_sec = 0;
1104 v.tm.tv_usec = 0;
1105 } else {
1106 v.tm.tv_sec = sk->sk_sndtimeo / HZ;
1107 v.tm.tv_usec = ((sk->sk_sndtimeo % HZ) * 1000000) / HZ;
1108 }
1109 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001110
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001111 case SO_RCVLOWAT:
1112 v.val = sk->sk_rcvlowat;
1113 break;
Catherine Zhang877ce7c2006-06-29 12:27:47 -07001114
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001115 case SO_SNDLOWAT:
Eric Dumazet2a915252009-05-27 11:30:05 +00001116 v.val = 1;
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001117 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001118
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001119 case SO_PASSCRED:
Eric Dumazet82981932012-04-26 20:07:59 +00001120 v.val = !!test_bit(SOCK_PASSCRED, &sock->flags);
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001121 break;
1122
1123 case SO_PEERCRED:
Eric W. Biederman109f6e32010-06-13 03:30:14 +00001124 {
1125 struct ucred peercred;
1126 if (len > sizeof(peercred))
1127 len = sizeof(peercred);
1128 cred_to_ucred(sk->sk_peer_pid, sk->sk_peer_cred, &peercred);
1129 if (copy_to_user(optval, &peercred, len))
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001130 return -EFAULT;
1131 goto lenout;
Eric W. Biederman109f6e32010-06-13 03:30:14 +00001132 }
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001133
1134 case SO_PEERNAME:
1135 {
1136 char address[128];
1137
1138 if (sock->ops->getname(sock, (struct sockaddr *)address, &lv, 2))
1139 return -ENOTCONN;
1140 if (lv < len)
1141 return -EINVAL;
1142 if (copy_to_user(optval, address, len))
1143 return -EFAULT;
1144 goto lenout;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001145 }
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001146
1147 /* Dubious BSD thing... Probably nobody even uses it, but
1148 * the UNIX standard wants it for whatever reason... -DaveM
1149 */
1150 case SO_ACCEPTCONN:
1151 v.val = sk->sk_state == TCP_LISTEN;
1152 break;
1153
1154 case SO_PASSSEC:
Eric Dumazet82981932012-04-26 20:07:59 +00001155 v.val = !!test_bit(SOCK_PASSSEC, &sock->flags);
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001156 break;
1157
1158 case SO_PEERSEC:
1159 return security_socket_getpeersec_stream(sock, optval, optlen, len);
1160
Laszlo Attila Toth4a19ec52008-01-30 19:08:16 -08001161 case SO_MARK:
1162 v.val = sk->sk_mark;
1163 break;
1164
Neil Horman3b885782009-10-12 13:26:31 -07001165 case SO_RXQ_OVFL:
Eric Dumazet1b23a5d2012-05-16 05:57:07 +00001166 v.val = sock_flag(sk, SOCK_RXQ_OVFL);
Neil Horman3b885782009-10-12 13:26:31 -07001167 break;
1168
Johannes Berg6e3e9392011-11-09 10:15:42 +01001169 case SO_WIFI_STATUS:
Eric Dumazet1b23a5d2012-05-16 05:57:07 +00001170 v.val = sock_flag(sk, SOCK_WIFI_STATUS);
Johannes Berg6e3e9392011-11-09 10:15:42 +01001171 break;
1172
Pavel Emelyanovef64a542012-02-21 07:31:34 +00001173 case SO_PEEK_OFF:
1174 if (!sock->ops->set_peek_off)
1175 return -EOPNOTSUPP;
1176
1177 v.val = sk->sk_peek_off;
1178 break;
David S. Millerbc2f7992012-02-24 14:48:34 -05001179 case SO_NOFCS:
Eric Dumazet1b23a5d2012-05-16 05:57:07 +00001180 v.val = sock_flag(sk, SOCK_NOFCS);
David S. Millerbc2f7992012-02-24 14:48:34 -05001181 break;
Brian Haleyc91f6df2012-11-26 05:21:08 +00001182
Pavel Emelyanovf7b86bf2012-10-18 23:55:56 +00001183 case SO_BINDTODEVICE:
Brian Haleyc91f6df2012-11-26 05:21:08 +00001184 return sock_getbindtodevice(sk, optval, optlen, len);
1185
Pavel Emelyanova8fc9272012-11-01 02:01:48 +00001186 case SO_GET_FILTER:
1187 len = sk_get_filter(sk, (struct sock_filter __user *)optval, len);
1188 if (len < 0)
1189 return len;
1190
1191 goto lenout;
Brian Haleyc91f6df2012-11-26 05:21:08 +00001192
Vincent Bernatd59577b2013-01-16 22:55:49 +01001193 case SO_LOCK_FILTER:
1194 v.val = sock_flag(sk, SOCK_FILTER_LOCKED);
1195 break;
1196
Michal Sekletarea02f942014-01-17 17:09:45 +01001197 case SO_BPF_EXTENSIONS:
1198 v.val = bpf_tell_extensions();
1199 break;
1200
Keller, Jacob E7d4c04f2013-03-28 11:19:25 +00001201 case SO_SELECT_ERR_QUEUE:
1202 v.val = sock_flag(sk, SOCK_SELECT_ERR_QUEUE);
1203 break;
1204
Cong Wange0d10952013-08-01 11:10:25 +08001205#ifdef CONFIG_NET_RX_BUSY_POLL
Eliezer Tamir64b0dc52013-07-10 17:13:36 +03001206 case SO_BUSY_POLL:
Eliezer Tamirdafcc432013-06-14 16:33:57 +03001207 v.val = sk->sk_ll_usec;
1208 break;
1209#endif
1210
Eric Dumazet62748f32013-09-24 08:20:52 -07001211 case SO_MAX_PACING_RATE:
1212 v.val = sk->sk_max_pacing_rate;
1213 break;
1214
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001215 default:
1216 return -ENOPROTOOPT;
1217 }
1218
Linus Torvalds1da177e2005-04-16 15:20:36 -07001219 if (len > lv)
1220 len = lv;
1221 if (copy_to_user(optval, &v, len))
1222 return -EFAULT;
1223lenout:
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001224 if (put_user(len, optlen))
1225 return -EFAULT;
1226 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001227}
1228
Ingo Molnara5b5bb92006-07-03 00:25:35 -07001229/*
1230 * Initialize an sk_lock.
1231 *
1232 * (We also register the sk_lock with the lock validator.)
1233 */
Dave Jonesb6f99a22007-03-22 12:27:49 -07001234static inline void sock_lock_init(struct sock *sk)
Ingo Molnara5b5bb92006-07-03 00:25:35 -07001235{
Peter Zijlstraed075362006-12-06 20:35:24 -08001236 sock_lock_init_class_and_name(sk,
1237 af_family_slock_key_strings[sk->sk_family],
1238 af_family_slock_keys + sk->sk_family,
1239 af_family_key_strings[sk->sk_family],
1240 af_family_keys + sk->sk_family);
Ingo Molnara5b5bb92006-07-03 00:25:35 -07001241}
1242
Eric Dumazet4dc6dc72009-07-15 23:13:10 +00001243/*
1244 * Copy all fields from osk to nsk but nsk->sk_refcnt must not change yet,
1245 * even temporarly, because of RCU lookups. sk_node should also be left as is.
Eric Dumazet68835ab2010-11-30 19:04:07 +00001246 * We must not copy fields between sk_dontcopy_begin and sk_dontcopy_end
Eric Dumazet4dc6dc72009-07-15 23:13:10 +00001247 */
Pavel Emelyanovf1a6c4d2007-11-01 00:29:45 -07001248static void sock_copy(struct sock *nsk, const struct sock *osk)
1249{
1250#ifdef CONFIG_SECURITY_NETWORK
1251 void *sptr = nsk->sk_security;
1252#endif
Eric Dumazet68835ab2010-11-30 19:04:07 +00001253 memcpy(nsk, osk, offsetof(struct sock, sk_dontcopy_begin));
1254
1255 memcpy(&nsk->sk_dontcopy_end, &osk->sk_dontcopy_end,
1256 osk->sk_prot->obj_size - offsetof(struct sock, sk_dontcopy_end));
1257
Pavel Emelyanovf1a6c4d2007-11-01 00:29:45 -07001258#ifdef CONFIG_SECURITY_NETWORK
1259 nsk->sk_security = sptr;
1260 security_sk_clone(osk, nsk);
1261#endif
1262}
1263
Octavian Purdilafcbdf092010-12-16 14:26:56 -08001264void sk_prot_clear_portaddr_nulls(struct sock *sk, int size)
1265{
1266 unsigned long nulls1, nulls2;
1267
1268 nulls1 = offsetof(struct sock, __sk_common.skc_node.next);
1269 nulls2 = offsetof(struct sock, __sk_common.skc_portaddr_node.next);
1270 if (nulls1 > nulls2)
1271 swap(nulls1, nulls2);
1272
1273 if (nulls1 != 0)
1274 memset((char *)sk, 0, nulls1);
1275 memset((char *)sk + nulls1 + sizeof(void *), 0,
1276 nulls2 - nulls1 - sizeof(void *));
1277 memset((char *)sk + nulls2 + sizeof(void *), 0,
1278 size - nulls2 - sizeof(void *));
1279}
1280EXPORT_SYMBOL(sk_prot_clear_portaddr_nulls);
1281
Pavel Emelyanov2e4afe72007-11-01 00:36:26 -07001282static struct sock *sk_prot_alloc(struct proto *prot, gfp_t priority,
1283 int family)
Pavel Emelyanovc308c1b22007-11-01 00:33:50 -07001284{
1285 struct sock *sk;
1286 struct kmem_cache *slab;
1287
1288 slab = prot->slab;
Eric Dumazete912b112009-07-08 19:36:05 +00001289 if (slab != NULL) {
1290 sk = kmem_cache_alloc(slab, priority & ~__GFP_ZERO);
1291 if (!sk)
1292 return sk;
1293 if (priority & __GFP_ZERO) {
Octavian Purdilafcbdf092010-12-16 14:26:56 -08001294 if (prot->clear_sk)
1295 prot->clear_sk(sk, prot->obj_size);
1296 else
1297 sk_prot_clear_nulls(sk, prot->obj_size);
Eric Dumazete912b112009-07-08 19:36:05 +00001298 }
Octavian Purdilafcbdf092010-12-16 14:26:56 -08001299 } else
Pavel Emelyanovc308c1b22007-11-01 00:33:50 -07001300 sk = kmalloc(prot->obj_size, priority);
1301
Pavel Emelyanov2e4afe72007-11-01 00:36:26 -07001302 if (sk != NULL) {
Vegard Nossuma98b65a2009-02-26 14:46:57 +01001303 kmemcheck_annotate_bitfield(sk, flags);
1304
Pavel Emelyanov2e4afe72007-11-01 00:36:26 -07001305 if (security_sk_alloc(sk, family, priority))
1306 goto out_free;
1307
1308 if (!try_module_get(prot->owner))
1309 goto out_free_sec;
Krishna Kumare022f0b2009-10-19 23:46:20 +00001310 sk_tx_queue_clear(sk);
Pavel Emelyanov2e4afe72007-11-01 00:36:26 -07001311 }
1312
Pavel Emelyanovc308c1b22007-11-01 00:33:50 -07001313 return sk;
Pavel Emelyanov2e4afe72007-11-01 00:36:26 -07001314
1315out_free_sec:
1316 security_sk_free(sk);
1317out_free:
1318 if (slab != NULL)
1319 kmem_cache_free(slab, sk);
1320 else
1321 kfree(sk);
1322 return NULL;
Pavel Emelyanovc308c1b22007-11-01 00:33:50 -07001323}
1324
1325static void sk_prot_free(struct proto *prot, struct sock *sk)
1326{
1327 struct kmem_cache *slab;
Pavel Emelyanov2e4afe72007-11-01 00:36:26 -07001328 struct module *owner;
Pavel Emelyanovc308c1b22007-11-01 00:33:50 -07001329
Pavel Emelyanov2e4afe72007-11-01 00:36:26 -07001330 owner = prot->owner;
Pavel Emelyanovc308c1b22007-11-01 00:33:50 -07001331 slab = prot->slab;
Pavel Emelyanov2e4afe72007-11-01 00:36:26 -07001332
1333 security_sk_free(sk);
Pavel Emelyanovc308c1b22007-11-01 00:33:50 -07001334 if (slab != NULL)
1335 kmem_cache_free(slab, sk);
1336 else
1337 kfree(sk);
Pavel Emelyanov2e4afe72007-11-01 00:36:26 -07001338 module_put(owner);
Pavel Emelyanovc308c1b22007-11-01 00:33:50 -07001339}
1340
Daniel Borkmann86f85152013-12-29 17:27:11 +01001341#if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
Zefan Li6ffd4642013-04-08 20:03:47 +00001342void sock_update_netprioidx(struct sock *sk)
Neil Horman5bc14212011-11-22 05:10:51 +00001343{
Neil Horman5bc14212011-11-22 05:10:51 +00001344 if (in_interrupt())
1345 return;
Neil Horman2b73bc62012-02-10 05:43:38 +00001346
Zefan Li6ffd4642013-04-08 20:03:47 +00001347 sk->sk_cgrp_prioidx = task_netprioidx(current);
Neil Horman5bc14212011-11-22 05:10:51 +00001348}
1349EXPORT_SYMBOL_GPL(sock_update_netprioidx);
Herbert Xuf8451722010-05-24 00:12:34 -07001350#endif
1351
Linus Torvalds1da177e2005-04-16 15:20:36 -07001352/**
1353 * sk_alloc - All socket objects are allocated here
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07001354 * @net: the applicable net namespace
Pavel Pisa4dc3b162005-05-01 08:59:25 -07001355 * @family: protocol family
1356 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc)
1357 * @prot: struct proto associated with this new sock instance
Linus Torvalds1da177e2005-04-16 15:20:36 -07001358 */
Eric W. Biederman1b8d7ae2007-10-08 23:24:22 -07001359struct sock *sk_alloc(struct net *net, int family, gfp_t priority,
Pavel Emelyanov6257ff22007-11-01 00:39:31 -07001360 struct proto *prot)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001361{
Pavel Emelyanovc308c1b22007-11-01 00:33:50 -07001362 struct sock *sk;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001363
Pavel Emelyanov154adbc2007-11-01 00:38:43 -07001364 sk = sk_prot_alloc(prot, priority | __GFP_ZERO, family);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001365 if (sk) {
Pavel Emelyanov154adbc2007-11-01 00:38:43 -07001366 sk->sk_family = family;
1367 /*
1368 * See comment in struct sock definition to understand
1369 * why we need sk_prot_creator -acme
1370 */
1371 sk->sk_prot = sk->sk_prot_creator = prot;
1372 sock_lock_init(sk);
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09001373 sock_net_set(sk, get_net(net));
Jarek Poplawskid66ee052009-08-30 23:15:36 +00001374 atomic_set(&sk->sk_wmem_alloc, 1);
Herbert Xuf8451722010-05-24 00:12:34 -07001375
Zefan Li211d2f972013-04-08 20:03:35 +00001376 sock_update_classid(sk);
Zefan Li6ffd4642013-04-08 20:03:47 +00001377 sock_update_netprioidx(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001378 }
Frank Filza79af592005-09-27 15:23:38 -07001379
Pavel Emelyanov2e4afe72007-11-01 00:36:26 -07001380 return sk;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001381}
Eric Dumazet2a915252009-05-27 11:30:05 +00001382EXPORT_SYMBOL(sk_alloc);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001383
Eric Dumazet2b85a342009-06-11 02:55:43 -07001384static void __sk_free(struct sock *sk)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001385{
1386 struct sk_filter *filter;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001387
1388 if (sk->sk_destruct)
1389 sk->sk_destruct(sk);
1390
Paul E. McKenneya898def2010-02-22 17:04:49 -08001391 filter = rcu_dereference_check(sk->sk_filter,
1392 atomic_read(&sk->sk_wmem_alloc) == 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001393 if (filter) {
Pavel Emelyanov309dd5f2007-10-17 21:21:51 -07001394 sk_filter_uncharge(sk, filter);
Stephen Hemmingera9b3cd72011-08-01 16:19:00 +00001395 RCU_INIT_POINTER(sk->sk_filter, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001396 }
1397
Eric Dumazet08e29af2011-11-28 12:04:18 +00001398 sock_disable_timestamp(sk, SK_FLAGS_TIMESTAMP);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001399
1400 if (atomic_read(&sk->sk_omem_alloc))
Joe Perchese005d192012-05-16 19:58:40 +00001401 pr_debug("%s: optmem leakage (%d bytes) detected\n",
1402 __func__, atomic_read(&sk->sk_omem_alloc));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001403
Eric W. Biederman109f6e32010-06-13 03:30:14 +00001404 if (sk->sk_peer_cred)
1405 put_cred(sk->sk_peer_cred);
1406 put_pid(sk->sk_peer_pid);
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09001407 put_net(sock_net(sk));
Pavel Emelyanovc308c1b22007-11-01 00:33:50 -07001408 sk_prot_free(sk->sk_prot_creator, sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001409}
Eric Dumazet2b85a342009-06-11 02:55:43 -07001410
1411void sk_free(struct sock *sk)
1412{
1413 /*
Lucas De Marchi25985ed2011-03-30 22:57:33 -03001414 * We subtract one from sk_wmem_alloc and can know if
Eric Dumazet2b85a342009-06-11 02:55:43 -07001415 * some packets are still in some tx queue.
1416 * If not null, sock_wfree() will call __sk_free(sk) later
1417 */
1418 if (atomic_dec_and_test(&sk->sk_wmem_alloc))
1419 __sk_free(sk);
1420}
Eric Dumazet2a915252009-05-27 11:30:05 +00001421EXPORT_SYMBOL(sk_free);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001422
Denis V. Lunevedf02082008-02-29 11:18:32 -08001423/*
Lucas De Marchi25985ed2011-03-30 22:57:33 -03001424 * Last sock_put should drop reference to sk->sk_net. It has already
1425 * been dropped in sk_change_net. Taking reference to stopping namespace
Denis V. Lunevedf02082008-02-29 11:18:32 -08001426 * is not an option.
Lucas De Marchi25985ed2011-03-30 22:57:33 -03001427 * Take reference to a socket to remove it from hash _alive_ and after that
Denis V. Lunevedf02082008-02-29 11:18:32 -08001428 * destroy it in the context of init_net.
1429 */
1430void sk_release_kernel(struct sock *sk)
1431{
1432 if (sk == NULL || sk->sk_socket == NULL)
1433 return;
1434
1435 sock_hold(sk);
1436 sock_release(sk->sk_socket);
Denis V. Lunev65a18ec2008-04-16 01:59:46 -07001437 release_net(sock_net(sk));
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09001438 sock_net_set(sk, get_net(&init_net));
Denis V. Lunevedf02082008-02-29 11:18:32 -08001439 sock_put(sk);
1440}
David S. Miller45af1752008-02-29 11:33:19 -08001441EXPORT_SYMBOL(sk_release_kernel);
Denis V. Lunevedf02082008-02-29 11:18:32 -08001442
Stephen Rothwell475f1b52012-01-09 16:33:16 +11001443static void sk_update_clone(const struct sock *sk, struct sock *newsk)
1444{
1445 if (mem_cgroup_sockets_enabled && sk->sk_cgrp)
1446 sock_update_memcg(newsk);
1447}
1448
Eric Dumazete56c57d2011-11-08 17:07:07 -05001449/**
1450 * sk_clone_lock - clone a socket, and lock its clone
1451 * @sk: the socket to clone
1452 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc)
1453 *
1454 * Caller must unlock socket even in error path (bh_unlock_sock(newsk))
1455 */
1456struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority)
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001457{
Pavel Emelyanov8fd1d172007-11-01 00:37:32 -07001458 struct sock *newsk;
Alexei Starovoitov278571b2014-07-30 20:34:12 -07001459 bool is_charged = true;
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001460
Pavel Emelyanov8fd1d172007-11-01 00:37:32 -07001461 newsk = sk_prot_alloc(sk->sk_prot, priority, sk->sk_family);
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001462 if (newsk != NULL) {
1463 struct sk_filter *filter;
1464
Venkat Yekkirala892c1412006-08-04 23:08:56 -07001465 sock_copy(newsk, sk);
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001466
1467 /* SANITY */
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09001468 get_net(sock_net(newsk));
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001469 sk_node_init(&newsk->sk_node);
1470 sock_lock_init(newsk);
1471 bh_lock_sock(newsk);
Eric Dumazetfa438cc2007-03-04 16:05:44 -08001472 newsk->sk_backlog.head = newsk->sk_backlog.tail = NULL;
Zhu Yi8eae9392010-03-04 18:01:40 +00001473 newsk->sk_backlog.len = 0;
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001474
1475 atomic_set(&newsk->sk_rmem_alloc, 0);
Eric Dumazet2b85a342009-06-11 02:55:43 -07001476 /*
1477 * sk_wmem_alloc set to one (see sk_free() and sock_wfree())
1478 */
1479 atomic_set(&newsk->sk_wmem_alloc, 1);
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001480 atomic_set(&newsk->sk_omem_alloc, 0);
1481 skb_queue_head_init(&newsk->sk_receive_queue);
1482 skb_queue_head_init(&newsk->sk_write_queue);
Chris Leech97fc2f02006-05-23 17:55:33 -07001483#ifdef CONFIG_NET_DMA
1484 skb_queue_head_init(&newsk->sk_async_wait_queue);
1485#endif
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001486
Eric Dumazetb6c67122010-04-08 23:03:29 +00001487 spin_lock_init(&newsk->sk_dst_lock);
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001488 rwlock_init(&newsk->sk_callback_lock);
Peter Zijlstra443aef0e2007-07-19 01:49:00 -07001489 lockdep_set_class_and_name(&newsk->sk_callback_lock,
1490 af_callback_keys + newsk->sk_family,
1491 af_family_clock_key_strings[newsk->sk_family]);
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001492
1493 newsk->sk_dst_cache = NULL;
1494 newsk->sk_wmem_queued = 0;
1495 newsk->sk_forward_alloc = 0;
1496 newsk->sk_send_head = NULL;
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001497 newsk->sk_userlocks = sk->sk_userlocks & ~SOCK_BINDPORT_LOCK;
1498
1499 sock_reset_flag(newsk, SOCK_DONE);
1500 skb_queue_head_init(&newsk->sk_error_queue);
1501
Eric Dumazet0d7da9d2010-10-25 03:47:05 +00001502 filter = rcu_dereference_protected(newsk->sk_filter, 1);
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001503 if (filter != NULL)
Alexei Starovoitov278571b2014-07-30 20:34:12 -07001504 /* though it's an empty new sock, the charging may fail
1505 * if sysctl_optmem_max was changed between creation of
1506 * original socket and cloning
1507 */
1508 is_charged = sk_filter_charge(newsk, filter);
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001509
Alexei Starovoitov278571b2014-07-30 20:34:12 -07001510 if (unlikely(!is_charged || xfrm_sk_clone_policy(newsk))) {
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001511 /* It is still raw copy of parent, so invalidate
1512 * destructor and make plain sk_free() */
1513 newsk->sk_destruct = NULL;
Thomas Gleixnerb0691c82011-10-25 02:30:50 +00001514 bh_unlock_sock(newsk);
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001515 sk_free(newsk);
1516 newsk = NULL;
1517 goto out;
1518 }
1519
1520 newsk->sk_err = 0;
1521 newsk->sk_priority = 0;
Eric Dumazet4dc6dc72009-07-15 23:13:10 +00001522 /*
1523 * Before updating sk_refcnt, we must commit prior changes to memory
1524 * (Documentation/RCU/rculist_nulls.txt for details)
1525 */
1526 smp_wmb();
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001527 atomic_set(&newsk->sk_refcnt, 2);
1528
1529 /*
1530 * Increment the counter in the same struct proto as the master
1531 * sock (sk_refcnt_debug_inc uses newsk->sk_prot->socks, that
1532 * is the same as sk->sk_prot->socks, as this field was copied
1533 * with memcpy).
1534 *
1535 * This _changes_ the previous behaviour, where
1536 * tcp_create_openreq_child always was incrementing the
1537 * equivalent to tcp_prot->socks (inet_sock_nr), so this have
1538 * to be taken into account in all callers. -acme
1539 */
1540 sk_refcnt_debug_inc(newsk);
David S. Miller972692e2008-06-17 22:41:38 -07001541 sk_set_socket(newsk, NULL);
Eric Dumazet43815482010-04-29 11:01:49 +00001542 newsk->sk_wq = NULL;
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001543
Glauber Costaf3f511e2012-01-05 20:16:39 +00001544 sk_update_clone(sk, newsk);
1545
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001546 if (newsk->sk_prot->sockets_allocated)
Glauber Costa180d8cd2011-12-11 21:47:02 +00001547 sk_sockets_allocated_inc(newsk);
Octavian Purdila704da5602010-01-08 00:00:09 -08001548
Eric Dumazet08e29af2011-11-28 12:04:18 +00001549 if (newsk->sk_flags & SK_FLAGS_TIMESTAMP)
Octavian Purdila704da5602010-01-08 00:00:09 -08001550 net_enable_timestamp();
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001551 }
1552out:
1553 return newsk;
1554}
Eric Dumazete56c57d2011-11-08 17:07:07 -05001555EXPORT_SYMBOL_GPL(sk_clone_lock);
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001556
Andi Kleen99580892007-04-20 17:12:43 -07001557void sk_setup_caps(struct sock *sk, struct dst_entry *dst)
1558{
1559 __sk_dst_set(sk, dst);
1560 sk->sk_route_caps = dst->dev->features;
1561 if (sk->sk_route_caps & NETIF_F_GSO)
Herbert Xu4fcd6b92007-05-31 22:15:50 -07001562 sk->sk_route_caps |= NETIF_F_GSO_SOFTWARE;
Eric Dumazeta4654192010-05-16 00:36:33 -07001563 sk->sk_route_caps &= ~sk->sk_route_nocaps;
Andi Kleen99580892007-04-20 17:12:43 -07001564 if (sk_can_gso(sk)) {
Peter P Waskiewicz Jr82cc1a72008-03-21 03:43:19 -07001565 if (dst->header_len) {
Andi Kleen99580892007-04-20 17:12:43 -07001566 sk->sk_route_caps &= ~NETIF_F_GSO_MASK;
Peter P Waskiewicz Jr82cc1a72008-03-21 03:43:19 -07001567 } else {
Andi Kleen99580892007-04-20 17:12:43 -07001568 sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM;
Peter P Waskiewicz Jr82cc1a72008-03-21 03:43:19 -07001569 sk->sk_gso_max_size = dst->dev->gso_max_size;
Ben Hutchings14853482012-07-30 16:11:42 +00001570 sk->sk_gso_max_segs = dst->dev->gso_max_segs;
Peter P Waskiewicz Jr82cc1a72008-03-21 03:43:19 -07001571 }
Andi Kleen99580892007-04-20 17:12:43 -07001572 }
1573}
1574EXPORT_SYMBOL_GPL(sk_setup_caps);
1575
Linus Torvalds1da177e2005-04-16 15:20:36 -07001576/*
1577 * Simple resource managers for sockets.
1578 */
1579
1580
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001581/*
1582 * Write buffer destructor automatically called from kfree_skb.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001583 */
1584void sock_wfree(struct sk_buff *skb)
1585{
1586 struct sock *sk = skb->sk;
Eric Dumazetd99927f2009-09-24 10:49:24 +00001587 unsigned int len = skb->truesize;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001588
Eric Dumazetd99927f2009-09-24 10:49:24 +00001589 if (!sock_flag(sk, SOCK_USE_WRITE_QUEUE)) {
1590 /*
1591 * Keep a reference on sk_wmem_alloc, this will be released
1592 * after sk_write_space() call
1593 */
1594 atomic_sub(len - 1, &sk->sk_wmem_alloc);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001595 sk->sk_write_space(sk);
Eric Dumazetd99927f2009-09-24 10:49:24 +00001596 len = 1;
1597 }
Eric Dumazet2b85a342009-06-11 02:55:43 -07001598 /*
Eric Dumazetd99927f2009-09-24 10:49:24 +00001599 * if sk_wmem_alloc reaches 0, we must finish what sk_free()
1600 * could not do because of in-flight packets
Eric Dumazet2b85a342009-06-11 02:55:43 -07001601 */
Eric Dumazetd99927f2009-09-24 10:49:24 +00001602 if (atomic_sub_and_test(len, &sk->sk_wmem_alloc))
Eric Dumazet2b85a342009-06-11 02:55:43 -07001603 __sk_free(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001604}
Eric Dumazet2a915252009-05-27 11:30:05 +00001605EXPORT_SYMBOL(sock_wfree);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001606
Eric Dumazetf2f872f2013-07-30 17:55:08 -07001607void skb_orphan_partial(struct sk_buff *skb)
1608{
1609 /* TCP stack sets skb->ooo_okay based on sk_wmem_alloc,
1610 * so we do not completely orphan skb, but transfert all
1611 * accounted bytes but one, to avoid unexpected reorders.
1612 */
1613 if (skb->destructor == sock_wfree
1614#ifdef CONFIG_INET
1615 || skb->destructor == tcp_wfree
1616#endif
1617 ) {
1618 atomic_sub(skb->truesize - 1, &skb->sk->sk_wmem_alloc);
1619 skb->truesize = 1;
1620 } else {
1621 skb_orphan(skb);
1622 }
1623}
1624EXPORT_SYMBOL(skb_orphan_partial);
1625
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001626/*
1627 * Read buffer destructor automatically called from kfree_skb.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001628 */
1629void sock_rfree(struct sk_buff *skb)
1630{
1631 struct sock *sk = skb->sk;
Eric Dumazetd361fd52010-07-10 22:45:17 +00001632 unsigned int len = skb->truesize;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001633
Eric Dumazetd361fd52010-07-10 22:45:17 +00001634 atomic_sub(len, &sk->sk_rmem_alloc);
1635 sk_mem_uncharge(sk, len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001636}
Eric Dumazet2a915252009-05-27 11:30:05 +00001637EXPORT_SYMBOL(sock_rfree);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001638
David S. Miller41063e92012-06-19 21:22:05 -07001639void sock_edemux(struct sk_buff *skb)
1640{
Eric Dumazete8123472012-09-02 23:57:18 +00001641 struct sock *sk = skb->sk;
1642
Randy Dunlap1c463e52012-09-10 09:13:07 -07001643#ifdef CONFIG_INET
Eric Dumazete8123472012-09-02 23:57:18 +00001644 if (sk->sk_state == TCP_TIME_WAIT)
1645 inet_twsk_put(inet_twsk(sk));
1646 else
Randy Dunlap1c463e52012-09-10 09:13:07 -07001647#endif
Eric Dumazete8123472012-09-02 23:57:18 +00001648 sock_put(sk);
David S. Miller41063e92012-06-19 21:22:05 -07001649}
1650EXPORT_SYMBOL(sock_edemux);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001651
Eric W. Biederman976d02012012-05-23 17:16:53 -06001652kuid_t sock_i_uid(struct sock *sk)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001653{
Eric W. Biederman976d02012012-05-23 17:16:53 -06001654 kuid_t uid;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001655
Eric Dumazetf064af12010-09-22 12:43:39 +00001656 read_lock_bh(&sk->sk_callback_lock);
Eric W. Biederman976d02012012-05-23 17:16:53 -06001657 uid = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_uid : GLOBAL_ROOT_UID;
Eric Dumazetf064af12010-09-22 12:43:39 +00001658 read_unlock_bh(&sk->sk_callback_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001659 return uid;
1660}
Eric Dumazet2a915252009-05-27 11:30:05 +00001661EXPORT_SYMBOL(sock_i_uid);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001662
1663unsigned long sock_i_ino(struct sock *sk)
1664{
1665 unsigned long ino;
1666
Eric Dumazetf064af12010-09-22 12:43:39 +00001667 read_lock_bh(&sk->sk_callback_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001668 ino = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_ino : 0;
Eric Dumazetf064af12010-09-22 12:43:39 +00001669 read_unlock_bh(&sk->sk_callback_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001670 return ino;
1671}
Eric Dumazet2a915252009-05-27 11:30:05 +00001672EXPORT_SYMBOL(sock_i_ino);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001673
1674/*
1675 * Allocate a skb from the socket's send buffer.
1676 */
Victor Fusco86a76ca2005-07-08 14:57:47 -07001677struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force,
Al Virodd0fc662005-10-07 07:46:04 +01001678 gfp_t priority)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001679{
1680 if (force || atomic_read(&sk->sk_wmem_alloc) < sk->sk_sndbuf) {
Eric Dumazet2a915252009-05-27 11:30:05 +00001681 struct sk_buff *skb = alloc_skb(size, priority);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001682 if (skb) {
1683 skb_set_owner_w(skb, sk);
1684 return skb;
1685 }
1686 }
1687 return NULL;
1688}
Eric Dumazet2a915252009-05-27 11:30:05 +00001689EXPORT_SYMBOL(sock_wmalloc);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001690
1691/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001692 * Allocate a memory block from the socket's option memory buffer.
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001693 */
Al Virodd0fc662005-10-07 07:46:04 +01001694void *sock_kmalloc(struct sock *sk, int size, gfp_t priority)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001695{
Eric Dumazet95c96172012-04-15 05:58:06 +00001696 if ((unsigned int)size <= sysctl_optmem_max &&
Linus Torvalds1da177e2005-04-16 15:20:36 -07001697 atomic_read(&sk->sk_omem_alloc) + size < sysctl_optmem_max) {
1698 void *mem;
1699 /* First do the add, to avoid the race if kmalloc
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001700 * might sleep.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001701 */
1702 atomic_add(size, &sk->sk_omem_alloc);
1703 mem = kmalloc(size, priority);
1704 if (mem)
1705 return mem;
1706 atomic_sub(size, &sk->sk_omem_alloc);
1707 }
1708 return NULL;
1709}
Eric Dumazet2a915252009-05-27 11:30:05 +00001710EXPORT_SYMBOL(sock_kmalloc);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001711
1712/*
1713 * Free an option memory block.
1714 */
1715void sock_kfree_s(struct sock *sk, void *mem, int size)
1716{
1717 kfree(mem);
1718 atomic_sub(size, &sk->sk_omem_alloc);
1719}
Eric Dumazet2a915252009-05-27 11:30:05 +00001720EXPORT_SYMBOL(sock_kfree_s);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001721
1722/* It is almost wait_for_tcp_memory minus release_sock/lock_sock.
1723 I think, these locks should be removed for datagram sockets.
1724 */
Eric Dumazet2a915252009-05-27 11:30:05 +00001725static long sock_wait_for_wmem(struct sock *sk, long timeo)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001726{
1727 DEFINE_WAIT(wait);
1728
1729 clear_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags);
1730 for (;;) {
1731 if (!timeo)
1732 break;
1733 if (signal_pending(current))
1734 break;
1735 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
Eric Dumazetaa395142010-04-20 13:03:51 +00001736 prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001737 if (atomic_read(&sk->sk_wmem_alloc) < sk->sk_sndbuf)
1738 break;
1739 if (sk->sk_shutdown & SEND_SHUTDOWN)
1740 break;
1741 if (sk->sk_err)
1742 break;
1743 timeo = schedule_timeout(timeo);
1744 }
Eric Dumazetaa395142010-04-20 13:03:51 +00001745 finish_wait(sk_sleep(sk), &wait);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001746 return timeo;
1747}
1748
1749
1750/*
1751 * Generic send/receive buffer handlers
1752 */
1753
Herbert Xu4cc7f682009-02-04 16:55:54 -08001754struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len,
1755 unsigned long data_len, int noblock,
Eric Dumazet28d64272013-08-08 14:38:47 -07001756 int *errcode, int max_page_order)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001757{
Eric Dumazet28d64272013-08-08 14:38:47 -07001758 struct sk_buff *skb = NULL;
1759 unsigned long chunk;
Al Viro7d877f32005-10-21 03:20:43 -04001760 gfp_t gfp_mask;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001761 long timeo;
1762 int err;
Jason Wangcc9b17a2012-05-30 21:18:10 +00001763 int npages = (data_len + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
Eric Dumazet28d64272013-08-08 14:38:47 -07001764 struct page *page;
1765 int i;
Jason Wangcc9b17a2012-05-30 21:18:10 +00001766
1767 err = -EMSGSIZE;
1768 if (npages > MAX_SKB_FRAGS)
1769 goto failure;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001770
Linus Torvalds1da177e2005-04-16 15:20:36 -07001771 timeo = sock_sndtimeo(sk, noblock);
Eric Dumazet28d64272013-08-08 14:38:47 -07001772 while (!skb) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001773 err = sock_error(sk);
1774 if (err != 0)
1775 goto failure;
1776
1777 err = -EPIPE;
1778 if (sk->sk_shutdown & SEND_SHUTDOWN)
1779 goto failure;
1780
Eric Dumazet28d64272013-08-08 14:38:47 -07001781 if (atomic_read(&sk->sk_wmem_alloc) >= sk->sk_sndbuf) {
1782 set_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags);
1783 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1784 err = -EAGAIN;
1785 if (!timeo)
1786 goto failure;
1787 if (signal_pending(current))
1788 goto interrupted;
1789 timeo = sock_wait_for_wmem(sk, timeo);
1790 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001791 }
Eric Dumazet28d64272013-08-08 14:38:47 -07001792
1793 err = -ENOBUFS;
1794 gfp_mask = sk->sk_allocation;
1795 if (gfp_mask & __GFP_WAIT)
1796 gfp_mask |= __GFP_REPEAT;
1797
1798 skb = alloc_skb(header_len, gfp_mask);
1799 if (!skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001800 goto failure;
Eric Dumazet28d64272013-08-08 14:38:47 -07001801
1802 skb->truesize += data_len;
1803
1804 for (i = 0; npages > 0; i++) {
1805 int order = max_page_order;
1806
1807 while (order) {
1808 if (npages >= 1 << order) {
1809 page = alloc_pages(sk->sk_allocation |
Eric Dumazeted98df32014-02-06 10:42:42 -08001810 __GFP_COMP |
1811 __GFP_NOWARN |
1812 __GFP_NORETRY,
Eric Dumazet28d64272013-08-08 14:38:47 -07001813 order);
1814 if (page)
1815 goto fill_page;
1816 }
1817 order--;
1818 }
1819 page = alloc_page(sk->sk_allocation);
1820 if (!page)
1821 goto failure;
1822fill_page:
1823 chunk = min_t(unsigned long, data_len,
1824 PAGE_SIZE << order);
1825 skb_fill_page_desc(skb, i, page, 0, chunk);
1826 data_len -= chunk;
1827 npages -= 1 << order;
1828 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001829 }
1830
1831 skb_set_owner_w(skb, sk);
1832 return skb;
1833
1834interrupted:
1835 err = sock_intr_errno(timeo);
1836failure:
Eric Dumazet28d64272013-08-08 14:38:47 -07001837 kfree_skb(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001838 *errcode = err;
1839 return NULL;
1840}
Herbert Xu4cc7f682009-02-04 16:55:54 -08001841EXPORT_SYMBOL(sock_alloc_send_pskb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001842
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001843struct sk_buff *sock_alloc_send_skb(struct sock *sk, unsigned long size,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001844 int noblock, int *errcode)
1845{
Eric Dumazet28d64272013-08-08 14:38:47 -07001846 return sock_alloc_send_pskb(sk, size, 0, noblock, errcode, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001847}
Eric Dumazet2a915252009-05-27 11:30:05 +00001848EXPORT_SYMBOL(sock_alloc_send_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001849
Eric Dumazet5640f762012-09-23 23:04:42 +00001850/* On 32bit arches, an skb frag is limited to 2^15 */
1851#define SKB_FRAG_PAGE_ORDER get_order(32768)
1852
Eric Dumazet400dfd32013-10-17 16:27:07 -07001853/**
1854 * skb_page_frag_refill - check that a page_frag contains enough room
1855 * @sz: minimum size of the fragment we want to get
1856 * @pfrag: pointer to page_frag
1857 * @prio: priority for memory allocation
1858 *
1859 * Note: While this allocator tries to use high order pages, there is
1860 * no guarantee that allocations succeed. Therefore, @sz MUST be
1861 * less or equal than PAGE_SIZE.
1862 */
1863bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio)
Eric Dumazet5640f762012-09-23 23:04:42 +00001864{
1865 int order;
1866
1867 if (pfrag->page) {
1868 if (atomic_read(&pfrag->page->_count) == 1) {
1869 pfrag->offset = 0;
1870 return true;
1871 }
Eric Dumazet400dfd32013-10-17 16:27:07 -07001872 if (pfrag->offset + sz <= pfrag->size)
Eric Dumazet5640f762012-09-23 23:04:42 +00001873 return true;
1874 put_page(pfrag->page);
1875 }
1876
Michael Dalton097b4f12014-01-16 22:23:25 -08001877 order = SKB_FRAG_PAGE_ORDER;
Eric Dumazet5640f762012-09-23 23:04:42 +00001878 do {
Eric Dumazet400dfd32013-10-17 16:27:07 -07001879 gfp_t gfp = prio;
Eric Dumazet5640f762012-09-23 23:04:42 +00001880
1881 if (order)
Eric Dumazeted98df32014-02-06 10:42:42 -08001882 gfp |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY;
Eric Dumazet5640f762012-09-23 23:04:42 +00001883 pfrag->page = alloc_pages(gfp, order);
1884 if (likely(pfrag->page)) {
1885 pfrag->offset = 0;
1886 pfrag->size = PAGE_SIZE << order;
1887 return true;
1888 }
1889 } while (--order >= 0);
1890
Eric Dumazet400dfd32013-10-17 16:27:07 -07001891 return false;
1892}
1893EXPORT_SYMBOL(skb_page_frag_refill);
1894
1895bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag)
1896{
1897 if (likely(skb_page_frag_refill(32U, pfrag, sk->sk_allocation)))
1898 return true;
1899
Eric Dumazet5640f762012-09-23 23:04:42 +00001900 sk_enter_memory_pressure(sk);
1901 sk_stream_moderate_sndbuf(sk);
1902 return false;
1903}
1904EXPORT_SYMBOL(sk_page_frag_refill);
1905
Linus Torvalds1da177e2005-04-16 15:20:36 -07001906static void __lock_sock(struct sock *sk)
Namhyung Kimf39234d2010-09-08 03:48:48 +00001907 __releases(&sk->sk_lock.slock)
1908 __acquires(&sk->sk_lock.slock)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001909{
1910 DEFINE_WAIT(wait);
1911
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001912 for (;;) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001913 prepare_to_wait_exclusive(&sk->sk_lock.wq, &wait,
1914 TASK_UNINTERRUPTIBLE);
1915 spin_unlock_bh(&sk->sk_lock.slock);
1916 schedule();
1917 spin_lock_bh(&sk->sk_lock.slock);
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001918 if (!sock_owned_by_user(sk))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001919 break;
1920 }
1921 finish_wait(&sk->sk_lock.wq, &wait);
1922}
1923
1924static void __release_sock(struct sock *sk)
Namhyung Kimf39234d2010-09-08 03:48:48 +00001925 __releases(&sk->sk_lock.slock)
1926 __acquires(&sk->sk_lock.slock)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001927{
1928 struct sk_buff *skb = sk->sk_backlog.head;
1929
1930 do {
1931 sk->sk_backlog.head = sk->sk_backlog.tail = NULL;
1932 bh_unlock_sock(sk);
1933
1934 do {
1935 struct sk_buff *next = skb->next;
1936
Eric Dumazete4cbb022012-04-30 16:07:09 +00001937 prefetch(next);
Eric Dumazet7fee2262010-05-11 23:19:48 +00001938 WARN_ON_ONCE(skb_dst_is_noref(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001939 skb->next = NULL;
Peter Zijlstrac57943a2008-10-07 14:18:42 -07001940 sk_backlog_rcv(sk, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001941
1942 /*
1943 * We are in process context here with softirqs
1944 * disabled, use cond_resched_softirq() to preempt.
1945 * This is safe to do because we've taken the backlog
1946 * queue private:
1947 */
1948 cond_resched_softirq();
1949
1950 skb = next;
1951 } while (skb != NULL);
1952
1953 bh_lock_sock(sk);
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001954 } while ((skb = sk->sk_backlog.head) != NULL);
Zhu Yi8eae9392010-03-04 18:01:40 +00001955
1956 /*
1957 * Doing the zeroing here guarantee we can not loop forever
1958 * while a wild producer attempts to flood us.
1959 */
1960 sk->sk_backlog.len = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001961}
1962
1963/**
1964 * sk_wait_data - wait for data to arrive at sk_receive_queue
Pavel Pisa4dc3b162005-05-01 08:59:25 -07001965 * @sk: sock to wait on
1966 * @timeo: for how long
Linus Torvalds1da177e2005-04-16 15:20:36 -07001967 *
1968 * Now socket state including sk->sk_err is changed only under lock,
1969 * hence we may omit checks after joining wait queue.
1970 * We check receive queue before schedule() only as optimization;
1971 * it is very likely that release_sock() added new data.
1972 */
1973int sk_wait_data(struct sock *sk, long *timeo)
1974{
1975 int rc;
1976 DEFINE_WAIT(wait);
1977
Eric Dumazetaa395142010-04-20 13:03:51 +00001978 prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001979 set_bit(SOCK_ASYNC_WAITDATA, &sk->sk_socket->flags);
1980 rc = sk_wait_event(sk, timeo, !skb_queue_empty(&sk->sk_receive_queue));
1981 clear_bit(SOCK_ASYNC_WAITDATA, &sk->sk_socket->flags);
Eric Dumazetaa395142010-04-20 13:03:51 +00001982 finish_wait(sk_sleep(sk), &wait);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001983 return rc;
1984}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001985EXPORT_SYMBOL(sk_wait_data);
1986
Hideo Aoki3ab224b2007-12-31 00:11:19 -08001987/**
1988 * __sk_mem_schedule - increase sk_forward_alloc and memory_allocated
1989 * @sk: socket
1990 * @size: memory size to allocate
1991 * @kind: allocation type
1992 *
1993 * If kind is SK_MEM_SEND, it means wmem allocation. Otherwise it means
1994 * rmem allocation. This function assumes that protocols which have
1995 * memory_pressure use sk_wmem_queued as write buffer accounting.
1996 */
1997int __sk_mem_schedule(struct sock *sk, int size, int kind)
1998{
1999 struct proto *prot = sk->sk_prot;
2000 int amt = sk_mem_pages(size);
Eric Dumazet8d987e52010-11-09 23:24:26 +00002001 long allocated;
Glauber Costae1aab162011-12-11 21:47:03 +00002002 int parent_status = UNDER_LIMIT;
Hideo Aoki3ab224b2007-12-31 00:11:19 -08002003
2004 sk->sk_forward_alloc += amt * SK_MEM_QUANTUM;
Glauber Costa180d8cd2011-12-11 21:47:02 +00002005
Glauber Costae1aab162011-12-11 21:47:03 +00002006 allocated = sk_memory_allocated_add(sk, amt, &parent_status);
Hideo Aoki3ab224b2007-12-31 00:11:19 -08002007
2008 /* Under limit. */
Glauber Costae1aab162011-12-11 21:47:03 +00002009 if (parent_status == UNDER_LIMIT &&
2010 allocated <= sk_prot_mem_limits(sk, 0)) {
Glauber Costa180d8cd2011-12-11 21:47:02 +00002011 sk_leave_memory_pressure(sk);
Hideo Aoki3ab224b2007-12-31 00:11:19 -08002012 return 1;
2013 }
2014
Glauber Costae1aab162011-12-11 21:47:03 +00002015 /* Under pressure. (we or our parents) */
2016 if ((parent_status > SOFT_LIMIT) ||
2017 allocated > sk_prot_mem_limits(sk, 1))
Glauber Costa180d8cd2011-12-11 21:47:02 +00002018 sk_enter_memory_pressure(sk);
Hideo Aoki3ab224b2007-12-31 00:11:19 -08002019
Glauber Costae1aab162011-12-11 21:47:03 +00002020 /* Over hard limit (we or our parents) */
2021 if ((parent_status == OVER_LIMIT) ||
2022 (allocated > sk_prot_mem_limits(sk, 2)))
Hideo Aoki3ab224b2007-12-31 00:11:19 -08002023 goto suppress_allocation;
2024
2025 /* guarantee minimum buffer size under pressure */
2026 if (kind == SK_MEM_RECV) {
2027 if (atomic_read(&sk->sk_rmem_alloc) < prot->sysctl_rmem[0])
2028 return 1;
Glauber Costa180d8cd2011-12-11 21:47:02 +00002029
Hideo Aoki3ab224b2007-12-31 00:11:19 -08002030 } else { /* SK_MEM_SEND */
2031 if (sk->sk_type == SOCK_STREAM) {
2032 if (sk->sk_wmem_queued < prot->sysctl_wmem[0])
2033 return 1;
2034 } else if (atomic_read(&sk->sk_wmem_alloc) <
2035 prot->sysctl_wmem[0])
2036 return 1;
2037 }
2038
Glauber Costa180d8cd2011-12-11 21:47:02 +00002039 if (sk_has_memory_pressure(sk)) {
Eric Dumazet17483762008-11-25 21:16:35 -08002040 int alloc;
2041
Glauber Costa180d8cd2011-12-11 21:47:02 +00002042 if (!sk_under_memory_pressure(sk))
Eric Dumazet17483762008-11-25 21:16:35 -08002043 return 1;
Glauber Costa180d8cd2011-12-11 21:47:02 +00002044 alloc = sk_sockets_allocated_read_positive(sk);
2045 if (sk_prot_mem_limits(sk, 2) > alloc *
Hideo Aoki3ab224b2007-12-31 00:11:19 -08002046 sk_mem_pages(sk->sk_wmem_queued +
2047 atomic_read(&sk->sk_rmem_alloc) +
2048 sk->sk_forward_alloc))
2049 return 1;
2050 }
2051
2052suppress_allocation:
2053
2054 if (kind == SK_MEM_SEND && sk->sk_type == SOCK_STREAM) {
2055 sk_stream_moderate_sndbuf(sk);
2056
2057 /* Fail only if socket is _under_ its sndbuf.
2058 * In this case we cannot block, so that we have to fail.
2059 */
2060 if (sk->sk_wmem_queued + size >= sk->sk_sndbuf)
2061 return 1;
2062 }
2063
Satoru Moriya3847ce32011-06-17 12:00:03 +00002064 trace_sock_exceed_buf_limit(sk, prot, allocated);
2065
Hideo Aoki3ab224b2007-12-31 00:11:19 -08002066 /* Alas. Undo changes. */
2067 sk->sk_forward_alloc -= amt * SK_MEM_QUANTUM;
Glauber Costa180d8cd2011-12-11 21:47:02 +00002068
Glauber Costa0e90b312012-01-20 04:57:16 +00002069 sk_memory_allocated_sub(sk, amt);
Glauber Costa180d8cd2011-12-11 21:47:02 +00002070
Hideo Aoki3ab224b2007-12-31 00:11:19 -08002071 return 0;
2072}
Hideo Aoki3ab224b2007-12-31 00:11:19 -08002073EXPORT_SYMBOL(__sk_mem_schedule);
2074
2075/**
2076 * __sk_reclaim - reclaim memory_allocated
2077 * @sk: socket
2078 */
2079void __sk_mem_reclaim(struct sock *sk)
2080{
Glauber Costa180d8cd2011-12-11 21:47:02 +00002081 sk_memory_allocated_sub(sk,
Glauber Costa0e90b312012-01-20 04:57:16 +00002082 sk->sk_forward_alloc >> SK_MEM_QUANTUM_SHIFT);
Hideo Aoki3ab224b2007-12-31 00:11:19 -08002083 sk->sk_forward_alloc &= SK_MEM_QUANTUM - 1;
2084
Glauber Costa180d8cd2011-12-11 21:47:02 +00002085 if (sk_under_memory_pressure(sk) &&
2086 (sk_memory_allocated(sk) < sk_prot_mem_limits(sk, 0)))
2087 sk_leave_memory_pressure(sk);
Hideo Aoki3ab224b2007-12-31 00:11:19 -08002088}
Hideo Aoki3ab224b2007-12-31 00:11:19 -08002089EXPORT_SYMBOL(__sk_mem_reclaim);
2090
2091
Linus Torvalds1da177e2005-04-16 15:20:36 -07002092/*
2093 * Set of default routines for initialising struct proto_ops when
2094 * the protocol does not support a particular function. In certain
2095 * cases where it makes no sense for a protocol to have a "do nothing"
2096 * function, some default processing is provided.
2097 */
2098
2099int sock_no_bind(struct socket *sock, struct sockaddr *saddr, int len)
2100{
2101 return -EOPNOTSUPP;
2102}
Eric Dumazet2a915252009-05-27 11:30:05 +00002103EXPORT_SYMBOL(sock_no_bind);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002104
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002105int sock_no_connect(struct socket *sock, struct sockaddr *saddr,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002106 int len, int flags)
2107{
2108 return -EOPNOTSUPP;
2109}
Eric Dumazet2a915252009-05-27 11:30:05 +00002110EXPORT_SYMBOL(sock_no_connect);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002111
2112int sock_no_socketpair(struct socket *sock1, struct socket *sock2)
2113{
2114 return -EOPNOTSUPP;
2115}
Eric Dumazet2a915252009-05-27 11:30:05 +00002116EXPORT_SYMBOL(sock_no_socketpair);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002117
2118int sock_no_accept(struct socket *sock, struct socket *newsock, int flags)
2119{
2120 return -EOPNOTSUPP;
2121}
Eric Dumazet2a915252009-05-27 11:30:05 +00002122EXPORT_SYMBOL(sock_no_accept);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002123
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002124int sock_no_getname(struct socket *sock, struct sockaddr *saddr,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002125 int *len, int peer)
2126{
2127 return -EOPNOTSUPP;
2128}
Eric Dumazet2a915252009-05-27 11:30:05 +00002129EXPORT_SYMBOL(sock_no_getname);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002130
Eric Dumazet2a915252009-05-27 11:30:05 +00002131unsigned int sock_no_poll(struct file *file, struct socket *sock, poll_table *pt)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002132{
2133 return 0;
2134}
Eric Dumazet2a915252009-05-27 11:30:05 +00002135EXPORT_SYMBOL(sock_no_poll);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002136
2137int sock_no_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg)
2138{
2139 return -EOPNOTSUPP;
2140}
Eric Dumazet2a915252009-05-27 11:30:05 +00002141EXPORT_SYMBOL(sock_no_ioctl);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002142
2143int sock_no_listen(struct socket *sock, int backlog)
2144{
2145 return -EOPNOTSUPP;
2146}
Eric Dumazet2a915252009-05-27 11:30:05 +00002147EXPORT_SYMBOL(sock_no_listen);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002148
2149int sock_no_shutdown(struct socket *sock, int how)
2150{
2151 return -EOPNOTSUPP;
2152}
Eric Dumazet2a915252009-05-27 11:30:05 +00002153EXPORT_SYMBOL(sock_no_shutdown);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002154
2155int sock_no_setsockopt(struct socket *sock, int level, int optname,
David S. Millerb7058842009-09-30 16:12:20 -07002156 char __user *optval, unsigned int optlen)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002157{
2158 return -EOPNOTSUPP;
2159}
Eric Dumazet2a915252009-05-27 11:30:05 +00002160EXPORT_SYMBOL(sock_no_setsockopt);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002161
2162int sock_no_getsockopt(struct socket *sock, int level, int optname,
2163 char __user *optval, int __user *optlen)
2164{
2165 return -EOPNOTSUPP;
2166}
Eric Dumazet2a915252009-05-27 11:30:05 +00002167EXPORT_SYMBOL(sock_no_getsockopt);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002168
2169int sock_no_sendmsg(struct kiocb *iocb, struct socket *sock, struct msghdr *m,
2170 size_t len)
2171{
2172 return -EOPNOTSUPP;
2173}
Eric Dumazet2a915252009-05-27 11:30:05 +00002174EXPORT_SYMBOL(sock_no_sendmsg);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002175
2176int sock_no_recvmsg(struct kiocb *iocb, struct socket *sock, struct msghdr *m,
2177 size_t len, int flags)
2178{
2179 return -EOPNOTSUPP;
2180}
Eric Dumazet2a915252009-05-27 11:30:05 +00002181EXPORT_SYMBOL(sock_no_recvmsg);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002182
2183int sock_no_mmap(struct file *file, struct socket *sock, struct vm_area_struct *vma)
2184{
2185 /* Mirror missing mmap method error code */
2186 return -ENODEV;
2187}
Eric Dumazet2a915252009-05-27 11:30:05 +00002188EXPORT_SYMBOL(sock_no_mmap);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002189
2190ssize_t sock_no_sendpage(struct socket *sock, struct page *page, int offset, size_t size, int flags)
2191{
2192 ssize_t res;
2193 struct msghdr msg = {.msg_flags = flags};
2194 struct kvec iov;
2195 char *kaddr = kmap(page);
2196 iov.iov_base = kaddr + offset;
2197 iov.iov_len = size;
2198 res = kernel_sendmsg(sock, &msg, &iov, 1, size);
2199 kunmap(page);
2200 return res;
2201}
Eric Dumazet2a915252009-05-27 11:30:05 +00002202EXPORT_SYMBOL(sock_no_sendpage);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002203
2204/*
2205 * Default Socket Callbacks
2206 */
2207
2208static void sock_def_wakeup(struct sock *sk)
2209{
Eric Dumazet43815482010-04-29 11:01:49 +00002210 struct socket_wq *wq;
2211
2212 rcu_read_lock();
2213 wq = rcu_dereference(sk->sk_wq);
2214 if (wq_has_sleeper(wq))
2215 wake_up_interruptible_all(&wq->wait);
2216 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002217}
2218
2219static void sock_def_error_report(struct sock *sk)
2220{
Eric Dumazet43815482010-04-29 11:01:49 +00002221 struct socket_wq *wq;
2222
2223 rcu_read_lock();
2224 wq = rcu_dereference(sk->sk_wq);
2225 if (wq_has_sleeper(wq))
2226 wake_up_interruptible_poll(&wq->wait, POLLERR);
Pavel Emelyanov8d8ad9d2007-11-26 20:10:50 +08002227 sk_wake_async(sk, SOCK_WAKE_IO, POLL_ERR);
Eric Dumazet43815482010-04-29 11:01:49 +00002228 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002229}
2230
David S. Miller676d2362014-04-11 16:15:36 -04002231static void sock_def_readable(struct sock *sk)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002232{
Eric Dumazet43815482010-04-29 11:01:49 +00002233 struct socket_wq *wq;
2234
2235 rcu_read_lock();
2236 wq = rcu_dereference(sk->sk_wq);
2237 if (wq_has_sleeper(wq))
Eric Dumazet2c6607c2011-01-06 10:54:29 -08002238 wake_up_interruptible_sync_poll(&wq->wait, POLLIN | POLLPRI |
Davide Libenzi37e55402009-03-31 15:24:21 -07002239 POLLRDNORM | POLLRDBAND);
Pavel Emelyanov8d8ad9d2007-11-26 20:10:50 +08002240 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
Eric Dumazet43815482010-04-29 11:01:49 +00002241 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002242}
2243
2244static void sock_def_write_space(struct sock *sk)
2245{
Eric Dumazet43815482010-04-29 11:01:49 +00002246 struct socket_wq *wq;
2247
2248 rcu_read_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002249
2250 /* Do not wake up a writer until he can make "significant"
2251 * progress. --DaveM
2252 */
Stephen Hemmingere71a4782007-04-10 20:10:33 -07002253 if ((atomic_read(&sk->sk_wmem_alloc) << 1) <= sk->sk_sndbuf) {
Eric Dumazet43815482010-04-29 11:01:49 +00002254 wq = rcu_dereference(sk->sk_wq);
2255 if (wq_has_sleeper(wq))
2256 wake_up_interruptible_sync_poll(&wq->wait, POLLOUT |
Davide Libenzi37e55402009-03-31 15:24:21 -07002257 POLLWRNORM | POLLWRBAND);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002258
2259 /* Should agree with poll, otherwise some programs break */
2260 if (sock_writeable(sk))
Pavel Emelyanov8d8ad9d2007-11-26 20:10:50 +08002261 sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002262 }
2263
Eric Dumazet43815482010-04-29 11:01:49 +00002264 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002265}
2266
2267static void sock_def_destruct(struct sock *sk)
2268{
Jesper Juhla51482b2005-11-08 09:41:34 -08002269 kfree(sk->sk_protinfo);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002270}
2271
2272void sk_send_sigurg(struct sock *sk)
2273{
2274 if (sk->sk_socket && sk->sk_socket->file)
2275 if (send_sigurg(&sk->sk_socket->file->f_owner))
Pavel Emelyanov8d8ad9d2007-11-26 20:10:50 +08002276 sk_wake_async(sk, SOCK_WAKE_URG, POLL_PRI);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002277}
Eric Dumazet2a915252009-05-27 11:30:05 +00002278EXPORT_SYMBOL(sk_send_sigurg);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002279
2280void sk_reset_timer(struct sock *sk, struct timer_list* timer,
2281 unsigned long expires)
2282{
2283 if (!mod_timer(timer, expires))
2284 sock_hold(sk);
2285}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002286EXPORT_SYMBOL(sk_reset_timer);
2287
2288void sk_stop_timer(struct sock *sk, struct timer_list* timer)
2289{
Ying Xue25cc4ae2013-02-03 20:32:57 +00002290 if (del_timer(timer))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002291 __sock_put(sk);
2292}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002293EXPORT_SYMBOL(sk_stop_timer);
2294
2295void sock_init_data(struct socket *sock, struct sock *sk)
2296{
2297 skb_queue_head_init(&sk->sk_receive_queue);
2298 skb_queue_head_init(&sk->sk_write_queue);
2299 skb_queue_head_init(&sk->sk_error_queue);
Chris Leech97fc2f02006-05-23 17:55:33 -07002300#ifdef CONFIG_NET_DMA
2301 skb_queue_head_init(&sk->sk_async_wait_queue);
2302#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002303
2304 sk->sk_send_head = NULL;
2305
2306 init_timer(&sk->sk_timer);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002307
Linus Torvalds1da177e2005-04-16 15:20:36 -07002308 sk->sk_allocation = GFP_KERNEL;
2309 sk->sk_rcvbuf = sysctl_rmem_default;
2310 sk->sk_sndbuf = sysctl_wmem_default;
2311 sk->sk_state = TCP_CLOSE;
David S. Miller972692e2008-06-17 22:41:38 -07002312 sk_set_socket(sk, sock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002313
2314 sock_set_flag(sk, SOCK_ZAPPED);
2315
Stephen Hemmingere71a4782007-04-10 20:10:33 -07002316 if (sock) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002317 sk->sk_type = sock->type;
Eric Dumazet43815482010-04-29 11:01:49 +00002318 sk->sk_wq = sock->wq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002319 sock->sk = sk;
2320 } else
Eric Dumazet43815482010-04-29 11:01:49 +00002321 sk->sk_wq = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002322
Eric Dumazetb6c67122010-04-08 23:03:29 +00002323 spin_lock_init(&sk->sk_dst_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002324 rwlock_init(&sk->sk_callback_lock);
Peter Zijlstra443aef0e2007-07-19 01:49:00 -07002325 lockdep_set_class_and_name(&sk->sk_callback_lock,
2326 af_callback_keys + sk->sk_family,
2327 af_family_clock_key_strings[sk->sk_family]);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002328
2329 sk->sk_state_change = sock_def_wakeup;
2330 sk->sk_data_ready = sock_def_readable;
2331 sk->sk_write_space = sock_def_write_space;
2332 sk->sk_error_report = sock_def_error_report;
2333 sk->sk_destruct = sock_def_destruct;
2334
Eric Dumazet5640f762012-09-23 23:04:42 +00002335 sk->sk_frag.page = NULL;
2336 sk->sk_frag.offset = 0;
Pavel Emelyanovef64a542012-02-21 07:31:34 +00002337 sk->sk_peek_off = -1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002338
Eric W. Biederman109f6e32010-06-13 03:30:14 +00002339 sk->sk_peer_pid = NULL;
2340 sk->sk_peer_cred = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002341 sk->sk_write_pending = 0;
2342 sk->sk_rcvlowat = 1;
2343 sk->sk_rcvtimeo = MAX_SCHEDULE_TIMEOUT;
2344 sk->sk_sndtimeo = MAX_SCHEDULE_TIMEOUT;
2345
Eric Dumazetf37f0af2008-04-13 21:39:26 -07002346 sk->sk_stamp = ktime_set(-1L, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002347
Cong Wange0d10952013-08-01 11:10:25 +08002348#ifdef CONFIG_NET_RX_BUSY_POLL
Eliezer Tamir06021292013-06-10 11:39:50 +03002349 sk->sk_napi_id = 0;
Eliezer Tamir64b0dc52013-07-10 17:13:36 +03002350 sk->sk_ll_usec = sysctl_net_busy_read;
Eliezer Tamir06021292013-06-10 11:39:50 +03002351#endif
2352
Eric Dumazet62748f32013-09-24 08:20:52 -07002353 sk->sk_max_pacing_rate = ~0U;
Eric Dumazet7eec4172013-10-08 15:16:00 -07002354 sk->sk_pacing_rate = ~0U;
Eric Dumazet4dc6dc72009-07-15 23:13:10 +00002355 /*
2356 * Before updating sk_refcnt, we must commit prior changes to memory
2357 * (Documentation/RCU/rculist_nulls.txt for details)
2358 */
2359 smp_wmb();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002360 atomic_set(&sk->sk_refcnt, 1);
Wang Chen33c732c2007-11-13 20:30:01 -08002361 atomic_set(&sk->sk_drops, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002362}
Eric Dumazet2a915252009-05-27 11:30:05 +00002363EXPORT_SYMBOL(sock_init_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002364
Harvey Harrisonb5606c22008-02-13 15:03:16 -08002365void lock_sock_nested(struct sock *sk, int subclass)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002366{
2367 might_sleep();
Ingo Molnara5b5bb92006-07-03 00:25:35 -07002368 spin_lock_bh(&sk->sk_lock.slock);
John Heffnerd2e91172007-09-12 10:44:19 +02002369 if (sk->sk_lock.owned)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002370 __lock_sock(sk);
John Heffnerd2e91172007-09-12 10:44:19 +02002371 sk->sk_lock.owned = 1;
Ingo Molnara5b5bb92006-07-03 00:25:35 -07002372 spin_unlock(&sk->sk_lock.slock);
2373 /*
2374 * The sk_lock has mutex_lock() semantics here:
2375 */
Peter Zijlstrafcc70d52006-11-08 22:44:35 -08002376 mutex_acquire(&sk->sk_lock.dep_map, subclass, 0, _RET_IP_);
Ingo Molnara5b5bb92006-07-03 00:25:35 -07002377 local_bh_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002378}
Peter Zijlstrafcc70d52006-11-08 22:44:35 -08002379EXPORT_SYMBOL(lock_sock_nested);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002380
Harvey Harrisonb5606c22008-02-13 15:03:16 -08002381void release_sock(struct sock *sk)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002382{
Ingo Molnara5b5bb92006-07-03 00:25:35 -07002383 /*
2384 * The sk_lock has mutex_unlock() semantics:
2385 */
2386 mutex_release(&sk->sk_lock.dep_map, 1, _RET_IP_);
2387
2388 spin_lock_bh(&sk->sk_lock.slock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002389 if (sk->sk_backlog.tail)
2390 __release_sock(sk);
Eric Dumazet46d3cea2012-07-11 05:50:31 +00002391
Eric Dumazetc3f9b012014-03-10 09:50:11 -07002392 /* Warning : release_cb() might need to release sk ownership,
2393 * ie call sock_release_ownership(sk) before us.
2394 */
Eric Dumazet46d3cea2012-07-11 05:50:31 +00002395 if (sk->sk_prot->release_cb)
2396 sk->sk_prot->release_cb(sk);
2397
Eric Dumazetc3f9b012014-03-10 09:50:11 -07002398 sock_release_ownership(sk);
Ingo Molnara5b5bb92006-07-03 00:25:35 -07002399 if (waitqueue_active(&sk->sk_lock.wq))
2400 wake_up(&sk->sk_lock.wq);
2401 spin_unlock_bh(&sk->sk_lock.slock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002402}
2403EXPORT_SYMBOL(release_sock);
2404
Eric Dumazet8a74ad62010-05-26 19:20:18 +00002405/**
2406 * lock_sock_fast - fast version of lock_sock
2407 * @sk: socket
2408 *
2409 * This version should be used for very small section, where process wont block
2410 * return false if fast path is taken
2411 * sk_lock.slock locked, owned = 0, BH disabled
2412 * return true if slow path is taken
2413 * sk_lock.slock unlocked, owned = 1, BH enabled
2414 */
2415bool lock_sock_fast(struct sock *sk)
2416{
2417 might_sleep();
2418 spin_lock_bh(&sk->sk_lock.slock);
2419
2420 if (!sk->sk_lock.owned)
2421 /*
2422 * Note : We must disable BH
2423 */
2424 return false;
2425
2426 __lock_sock(sk);
2427 sk->sk_lock.owned = 1;
2428 spin_unlock(&sk->sk_lock.slock);
2429 /*
2430 * The sk_lock has mutex_lock() semantics here:
2431 */
2432 mutex_acquire(&sk->sk_lock.dep_map, 0, 0, _RET_IP_);
2433 local_bh_enable();
2434 return true;
2435}
2436EXPORT_SYMBOL(lock_sock_fast);
2437
Linus Torvalds1da177e2005-04-16 15:20:36 -07002438int sock_get_timestamp(struct sock *sk, struct timeval __user *userstamp)
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002439{
Eric Dumazetb7aa0bf2007-04-19 16:16:32 -07002440 struct timeval tv;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002441 if (!sock_flag(sk, SOCK_TIMESTAMP))
Patrick Ohly20d49472009-02-12 05:03:38 +00002442 sock_enable_timestamp(sk, SOCK_TIMESTAMP);
Eric Dumazetb7aa0bf2007-04-19 16:16:32 -07002443 tv = ktime_to_timeval(sk->sk_stamp);
2444 if (tv.tv_sec == -1)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002445 return -ENOENT;
Eric Dumazetb7aa0bf2007-04-19 16:16:32 -07002446 if (tv.tv_sec == 0) {
2447 sk->sk_stamp = ktime_get_real();
2448 tv = ktime_to_timeval(sk->sk_stamp);
2449 }
2450 return copy_to_user(userstamp, &tv, sizeof(tv)) ? -EFAULT : 0;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002451}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002452EXPORT_SYMBOL(sock_get_timestamp);
2453
Eric Dumazetae40eb12007-03-18 17:33:16 -07002454int sock_get_timestampns(struct sock *sk, struct timespec __user *userstamp)
2455{
2456 struct timespec ts;
2457 if (!sock_flag(sk, SOCK_TIMESTAMP))
Patrick Ohly20d49472009-02-12 05:03:38 +00002458 sock_enable_timestamp(sk, SOCK_TIMESTAMP);
Eric Dumazetae40eb12007-03-18 17:33:16 -07002459 ts = ktime_to_timespec(sk->sk_stamp);
2460 if (ts.tv_sec == -1)
2461 return -ENOENT;
2462 if (ts.tv_sec == 0) {
2463 sk->sk_stamp = ktime_get_real();
2464 ts = ktime_to_timespec(sk->sk_stamp);
2465 }
2466 return copy_to_user(userstamp, &ts, sizeof(ts)) ? -EFAULT : 0;
2467}
2468EXPORT_SYMBOL(sock_get_timestampns);
2469
Patrick Ohly20d49472009-02-12 05:03:38 +00002470void sock_enable_timestamp(struct sock *sk, int flag)
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002471{
Patrick Ohly20d49472009-02-12 05:03:38 +00002472 if (!sock_flag(sk, flag)) {
Eric Dumazet08e29af2011-11-28 12:04:18 +00002473 unsigned long previous_flags = sk->sk_flags;
2474
Patrick Ohly20d49472009-02-12 05:03:38 +00002475 sock_set_flag(sk, flag);
2476 /*
2477 * we just set one of the two flags which require net
2478 * time stamping, but time stamping might have been on
2479 * already because of the other one
2480 */
Eric Dumazet08e29af2011-11-28 12:04:18 +00002481 if (!(previous_flags & SK_FLAGS_TIMESTAMP))
Patrick Ohly20d49472009-02-12 05:03:38 +00002482 net_enable_timestamp();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002483 }
2484}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002485
Richard Cochrancb820f82013-07-19 19:40:09 +02002486int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len,
2487 int level, int type)
2488{
2489 struct sock_exterr_skb *serr;
2490 struct sk_buff *skb, *skb2;
2491 int copied, err;
2492
2493 err = -EAGAIN;
2494 skb = skb_dequeue(&sk->sk_error_queue);
2495 if (skb == NULL)
2496 goto out;
2497
2498 copied = skb->len;
2499 if (copied > len) {
2500 msg->msg_flags |= MSG_TRUNC;
2501 copied = len;
2502 }
2503 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
2504 if (err)
2505 goto out_free_skb;
2506
2507 sock_recv_timestamp(msg, sk, skb);
2508
2509 serr = SKB_EXT_ERR(skb);
2510 put_cmsg(msg, level, type, sizeof(serr->ee), &serr->ee);
2511
2512 msg->msg_flags |= MSG_ERRQUEUE;
2513 err = copied;
2514
2515 /* Reset and regenerate socket error */
2516 spin_lock_bh(&sk->sk_error_queue.lock);
2517 sk->sk_err = 0;
2518 if ((skb2 = skb_peek(&sk->sk_error_queue)) != NULL) {
2519 sk->sk_err = SKB_EXT_ERR(skb2)->ee.ee_errno;
2520 spin_unlock_bh(&sk->sk_error_queue.lock);
2521 sk->sk_error_report(sk);
2522 } else
2523 spin_unlock_bh(&sk->sk_error_queue.lock);
2524
2525out_free_skb:
2526 kfree_skb(skb);
2527out:
2528 return err;
2529}
2530EXPORT_SYMBOL(sock_recv_errqueue);
2531
Linus Torvalds1da177e2005-04-16 15:20:36 -07002532/*
2533 * Get a socket option on an socket.
2534 *
2535 * FIX: POSIX 1003.1g is very ambiguous here. It states that
2536 * asynchronous errors should be reported by getsockopt. We assume
2537 * this means if you specify SO_ERROR (otherwise whats the point of it).
2538 */
2539int sock_common_getsockopt(struct socket *sock, int level, int optname,
2540 char __user *optval, int __user *optlen)
2541{
2542 struct sock *sk = sock->sk;
2543
2544 return sk->sk_prot->getsockopt(sk, level, optname, optval, optlen);
2545}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002546EXPORT_SYMBOL(sock_common_getsockopt);
2547
Dmitry Mishin3fdadf72006-03-20 22:45:21 -08002548#ifdef CONFIG_COMPAT
Arnaldo Carvalho de Melo543d9cf2006-03-20 22:48:35 -08002549int compat_sock_common_getsockopt(struct socket *sock, int level, int optname,
2550 char __user *optval, int __user *optlen)
Dmitry Mishin3fdadf72006-03-20 22:45:21 -08002551{
2552 struct sock *sk = sock->sk;
2553
Johannes Berg1e51f952007-03-06 13:44:06 -08002554 if (sk->sk_prot->compat_getsockopt != NULL)
Arnaldo Carvalho de Melo543d9cf2006-03-20 22:48:35 -08002555 return sk->sk_prot->compat_getsockopt(sk, level, optname,
2556 optval, optlen);
Dmitry Mishin3fdadf72006-03-20 22:45:21 -08002557 return sk->sk_prot->getsockopt(sk, level, optname, optval, optlen);
2558}
2559EXPORT_SYMBOL(compat_sock_common_getsockopt);
2560#endif
2561
Linus Torvalds1da177e2005-04-16 15:20:36 -07002562int sock_common_recvmsg(struct kiocb *iocb, struct socket *sock,
2563 struct msghdr *msg, size_t size, int flags)
2564{
2565 struct sock *sk = sock->sk;
2566 int addr_len = 0;
2567 int err;
2568
2569 err = sk->sk_prot->recvmsg(iocb, sk, msg, size, flags & MSG_DONTWAIT,
2570 flags & ~MSG_DONTWAIT, &addr_len);
2571 if (err >= 0)
2572 msg->msg_namelen = addr_len;
2573 return err;
2574}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002575EXPORT_SYMBOL(sock_common_recvmsg);
2576
2577/*
2578 * Set socket options on an inet socket.
2579 */
2580int sock_common_setsockopt(struct socket *sock, int level, int optname,
David S. Millerb7058842009-09-30 16:12:20 -07002581 char __user *optval, unsigned int optlen)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002582{
2583 struct sock *sk = sock->sk;
2584
2585 return sk->sk_prot->setsockopt(sk, level, optname, optval, optlen);
2586}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002587EXPORT_SYMBOL(sock_common_setsockopt);
2588
Dmitry Mishin3fdadf72006-03-20 22:45:21 -08002589#ifdef CONFIG_COMPAT
Arnaldo Carvalho de Melo543d9cf2006-03-20 22:48:35 -08002590int compat_sock_common_setsockopt(struct socket *sock, int level, int optname,
David S. Millerb7058842009-09-30 16:12:20 -07002591 char __user *optval, unsigned int optlen)
Dmitry Mishin3fdadf72006-03-20 22:45:21 -08002592{
2593 struct sock *sk = sock->sk;
2594
Arnaldo Carvalho de Melo543d9cf2006-03-20 22:48:35 -08002595 if (sk->sk_prot->compat_setsockopt != NULL)
2596 return sk->sk_prot->compat_setsockopt(sk, level, optname,
2597 optval, optlen);
Dmitry Mishin3fdadf72006-03-20 22:45:21 -08002598 return sk->sk_prot->setsockopt(sk, level, optname, optval, optlen);
2599}
2600EXPORT_SYMBOL(compat_sock_common_setsockopt);
2601#endif
2602
Linus Torvalds1da177e2005-04-16 15:20:36 -07002603void sk_common_release(struct sock *sk)
2604{
2605 if (sk->sk_prot->destroy)
2606 sk->sk_prot->destroy(sk);
2607
2608 /*
2609 * Observation: when sock_common_release is called, processes have
2610 * no access to socket. But net still has.
2611 * Step one, detach it from networking:
2612 *
2613 * A. Remove from hash tables.
2614 */
2615
2616 sk->sk_prot->unhash(sk);
2617
2618 /*
2619 * In this point socket cannot receive new packets, but it is possible
2620 * that some packets are in flight because some CPU runs receiver and
2621 * did hash table lookup before we unhashed socket. They will achieve
2622 * receive queue and will be purged by socket destructor.
2623 *
2624 * Also we still have packets pending on receive queue and probably,
2625 * our own packets waiting in device queues. sock_destroy will drain
2626 * receive queue, but transmitted packets will delay socket destruction
2627 * until the last reference will be released.
2628 */
2629
2630 sock_orphan(sk);
2631
2632 xfrm_sk_free_policy(sk);
2633
Arnaldo Carvalho de Meloe6848972005-08-09 19:45:38 -07002634 sk_refcnt_debug_release(sk);
Eric Dumazet5640f762012-09-23 23:04:42 +00002635
2636 if (sk->sk_frag.page) {
2637 put_page(sk->sk_frag.page);
2638 sk->sk_frag.page = NULL;
2639 }
2640
Linus Torvalds1da177e2005-04-16 15:20:36 -07002641 sock_put(sk);
2642}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002643EXPORT_SYMBOL(sk_common_release);
2644
Pavel Emelyanov13ff3d62008-03-28 16:38:17 -07002645#ifdef CONFIG_PROC_FS
2646#define PROTO_INUSE_NR 64 /* should be enough for the first time */
Pavel Emelyanov1338d462008-03-28 16:38:43 -07002647struct prot_inuse {
2648 int val[PROTO_INUSE_NR];
2649};
Pavel Emelyanov13ff3d62008-03-28 16:38:17 -07002650
2651static DECLARE_BITMAP(proto_inuse_idx, PROTO_INUSE_NR);
Pavel Emelyanov70ee1152008-03-31 19:42:16 -07002652
2653#ifdef CONFIG_NET_NS
2654void sock_prot_inuse_add(struct net *net, struct proto *prot, int val)
2655{
Eric Dumazetd6d9ca02010-07-19 10:48:49 +00002656 __this_cpu_add(net->core.inuse->val[prot->inuse_idx], val);
Pavel Emelyanov70ee1152008-03-31 19:42:16 -07002657}
2658EXPORT_SYMBOL_GPL(sock_prot_inuse_add);
2659
2660int sock_prot_inuse_get(struct net *net, struct proto *prot)
2661{
2662 int cpu, idx = prot->inuse_idx;
2663 int res = 0;
2664
2665 for_each_possible_cpu(cpu)
2666 res += per_cpu_ptr(net->core.inuse, cpu)->val[idx];
2667
2668 return res >= 0 ? res : 0;
2669}
2670EXPORT_SYMBOL_GPL(sock_prot_inuse_get);
2671
Alexey Dobriyan2c8c1e72010-01-17 03:35:32 +00002672static int __net_init sock_inuse_init_net(struct net *net)
Pavel Emelyanov70ee1152008-03-31 19:42:16 -07002673{
2674 net->core.inuse = alloc_percpu(struct prot_inuse);
2675 return net->core.inuse ? 0 : -ENOMEM;
2676}
2677
Alexey Dobriyan2c8c1e72010-01-17 03:35:32 +00002678static void __net_exit sock_inuse_exit_net(struct net *net)
Pavel Emelyanov70ee1152008-03-31 19:42:16 -07002679{
2680 free_percpu(net->core.inuse);
2681}
2682
2683static struct pernet_operations net_inuse_ops = {
2684 .init = sock_inuse_init_net,
2685 .exit = sock_inuse_exit_net,
2686};
2687
2688static __init int net_inuse_init(void)
2689{
2690 if (register_pernet_subsys(&net_inuse_ops))
2691 panic("Cannot initialize net inuse counters");
2692
2693 return 0;
2694}
2695
2696core_initcall(net_inuse_init);
2697#else
Pavel Emelyanov1338d462008-03-28 16:38:43 -07002698static DEFINE_PER_CPU(struct prot_inuse, prot_inuse);
2699
Pavel Emelyanovc29a0bc2008-03-31 19:41:46 -07002700void sock_prot_inuse_add(struct net *net, struct proto *prot, int val)
Pavel Emelyanov1338d462008-03-28 16:38:43 -07002701{
Eric Dumazetd6d9ca02010-07-19 10:48:49 +00002702 __this_cpu_add(prot_inuse.val[prot->inuse_idx], val);
Pavel Emelyanov1338d462008-03-28 16:38:43 -07002703}
2704EXPORT_SYMBOL_GPL(sock_prot_inuse_add);
2705
Pavel Emelyanovc29a0bc2008-03-31 19:41:46 -07002706int sock_prot_inuse_get(struct net *net, struct proto *prot)
Pavel Emelyanov1338d462008-03-28 16:38:43 -07002707{
2708 int cpu, idx = prot->inuse_idx;
2709 int res = 0;
2710
2711 for_each_possible_cpu(cpu)
2712 res += per_cpu(prot_inuse, cpu).val[idx];
2713
2714 return res >= 0 ? res : 0;
2715}
2716EXPORT_SYMBOL_GPL(sock_prot_inuse_get);
Pavel Emelyanov70ee1152008-03-31 19:42:16 -07002717#endif
Pavel Emelyanov13ff3d62008-03-28 16:38:17 -07002718
2719static void assign_proto_idx(struct proto *prot)
2720{
2721 prot->inuse_idx = find_first_zero_bit(proto_inuse_idx, PROTO_INUSE_NR);
2722
2723 if (unlikely(prot->inuse_idx == PROTO_INUSE_NR - 1)) {
Joe Perchese005d192012-05-16 19:58:40 +00002724 pr_err("PROTO_INUSE_NR exhausted\n");
Pavel Emelyanov13ff3d62008-03-28 16:38:17 -07002725 return;
2726 }
2727
2728 set_bit(prot->inuse_idx, proto_inuse_idx);
2729}
2730
2731static void release_proto_idx(struct proto *prot)
2732{
2733 if (prot->inuse_idx != PROTO_INUSE_NR - 1)
2734 clear_bit(prot->inuse_idx, proto_inuse_idx);
2735}
2736#else
2737static inline void assign_proto_idx(struct proto *prot)
2738{
2739}
2740
2741static inline void release_proto_idx(struct proto *prot)
2742{
2743}
2744#endif
2745
Linus Torvalds1da177e2005-04-16 15:20:36 -07002746int proto_register(struct proto *prot, int alloc_slab)
2747{
Linus Torvalds1da177e2005-04-16 15:20:36 -07002748 if (alloc_slab) {
2749 prot->slab = kmem_cache_create(prot->name, prot->obj_size, 0,
Eric Dumazet271b72c2008-10-29 02:11:14 -07002750 SLAB_HWCACHE_ALIGN | prot->slab_flags,
2751 NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002752
2753 if (prot->slab == NULL) {
Joe Perchese005d192012-05-16 19:58:40 +00002754 pr_crit("%s: Can't create sock SLAB cache!\n",
2755 prot->name);
Pavel Emelyanov60e76632008-03-28 16:39:10 -07002756 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002757 }
Arnaldo Carvalho de Melo2e6599c2005-06-18 22:46:52 -07002758
2759 if (prot->rsk_prot != NULL) {
Alexey Dobriyanfaf23422010-02-17 09:34:12 +00002760 prot->rsk_prot->slab_name = kasprintf(GFP_KERNEL, "request_sock_%s", prot->name);
Catalin Marinas7e56b5d2008-11-21 16:45:22 -08002761 if (prot->rsk_prot->slab_name == NULL)
Arnaldo Carvalho de Melo2e6599c2005-06-18 22:46:52 -07002762 goto out_free_sock_slab;
2763
Catalin Marinas7e56b5d2008-11-21 16:45:22 -08002764 prot->rsk_prot->slab = kmem_cache_create(prot->rsk_prot->slab_name,
Arnaldo Carvalho de Melo2e6599c2005-06-18 22:46:52 -07002765 prot->rsk_prot->obj_size, 0,
Paul Mundt20c2df82007-07-20 10:11:58 +09002766 SLAB_HWCACHE_ALIGN, NULL);
Arnaldo Carvalho de Melo2e6599c2005-06-18 22:46:52 -07002767
2768 if (prot->rsk_prot->slab == NULL) {
Joe Perchese005d192012-05-16 19:58:40 +00002769 pr_crit("%s: Can't create request sock SLAB cache!\n",
2770 prot->name);
Arnaldo Carvalho de Melo2e6599c2005-06-18 22:46:52 -07002771 goto out_free_request_sock_slab_name;
2772 }
2773 }
Arnaldo Carvalho de Melo8feaf0c02005-08-09 20:09:30 -07002774
Arnaldo Carvalho de Melo6d6ee432005-12-13 23:25:19 -08002775 if (prot->twsk_prot != NULL) {
Alexey Dobriyanfaf23422010-02-17 09:34:12 +00002776 prot->twsk_prot->twsk_slab_name = kasprintf(GFP_KERNEL, "tw_sock_%s", prot->name);
Arnaldo Carvalho de Melo8feaf0c02005-08-09 20:09:30 -07002777
Catalin Marinas7e56b5d2008-11-21 16:45:22 -08002778 if (prot->twsk_prot->twsk_slab_name == NULL)
Arnaldo Carvalho de Melo8feaf0c02005-08-09 20:09:30 -07002779 goto out_free_request_sock_slab;
2780
Arnaldo Carvalho de Melo6d6ee432005-12-13 23:25:19 -08002781 prot->twsk_prot->twsk_slab =
Catalin Marinas7e56b5d2008-11-21 16:45:22 -08002782 kmem_cache_create(prot->twsk_prot->twsk_slab_name,
Arnaldo Carvalho de Melo6d6ee432005-12-13 23:25:19 -08002783 prot->twsk_prot->twsk_obj_size,
Eric Dumazet3ab5aee2008-11-16 19:40:17 -08002784 0,
2785 SLAB_HWCACHE_ALIGN |
2786 prot->slab_flags,
Paul Mundt20c2df82007-07-20 10:11:58 +09002787 NULL);
Arnaldo Carvalho de Melo6d6ee432005-12-13 23:25:19 -08002788 if (prot->twsk_prot->twsk_slab == NULL)
Arnaldo Carvalho de Melo8feaf0c02005-08-09 20:09:30 -07002789 goto out_free_timewait_sock_slab_name;
2790 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002791 }
2792
Glauber Costa36b77a52011-12-16 00:51:59 +00002793 mutex_lock(&proto_list_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002794 list_add(&prot->node, &proto_list);
Pavel Emelyanov13ff3d62008-03-28 16:38:17 -07002795 assign_proto_idx(prot);
Glauber Costa36b77a52011-12-16 00:51:59 +00002796 mutex_unlock(&proto_list_mutex);
Pavel Emelyanovb733c002007-11-07 02:23:38 -08002797 return 0;
2798
Arnaldo Carvalho de Melo8feaf0c02005-08-09 20:09:30 -07002799out_free_timewait_sock_slab_name:
Catalin Marinas7e56b5d2008-11-21 16:45:22 -08002800 kfree(prot->twsk_prot->twsk_slab_name);
Arnaldo Carvalho de Melo8feaf0c02005-08-09 20:09:30 -07002801out_free_request_sock_slab:
2802 if (prot->rsk_prot && prot->rsk_prot->slab) {
2803 kmem_cache_destroy(prot->rsk_prot->slab);
2804 prot->rsk_prot->slab = NULL;
2805 }
Arnaldo Carvalho de Melo2e6599c2005-06-18 22:46:52 -07002806out_free_request_sock_slab_name:
Dan Carpenter72150e92010-03-06 01:04:45 +00002807 if (prot->rsk_prot)
2808 kfree(prot->rsk_prot->slab_name);
Arnaldo Carvalho de Melo2e6599c2005-06-18 22:46:52 -07002809out_free_sock_slab:
2810 kmem_cache_destroy(prot->slab);
2811 prot->slab = NULL;
Pavel Emelyanovb733c002007-11-07 02:23:38 -08002812out:
2813 return -ENOBUFS;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002814}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002815EXPORT_SYMBOL(proto_register);
2816
2817void proto_unregister(struct proto *prot)
2818{
Glauber Costa36b77a52011-12-16 00:51:59 +00002819 mutex_lock(&proto_list_mutex);
Pavel Emelyanov13ff3d62008-03-28 16:38:17 -07002820 release_proto_idx(prot);
Patrick McHardy0a3f4352005-09-06 19:47:50 -07002821 list_del(&prot->node);
Glauber Costa36b77a52011-12-16 00:51:59 +00002822 mutex_unlock(&proto_list_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002823
2824 if (prot->slab != NULL) {
2825 kmem_cache_destroy(prot->slab);
2826 prot->slab = NULL;
2827 }
2828
Arnaldo Carvalho de Melo2e6599c2005-06-18 22:46:52 -07002829 if (prot->rsk_prot != NULL && prot->rsk_prot->slab != NULL) {
Arnaldo Carvalho de Melo2e6599c2005-06-18 22:46:52 -07002830 kmem_cache_destroy(prot->rsk_prot->slab);
Catalin Marinas7e56b5d2008-11-21 16:45:22 -08002831 kfree(prot->rsk_prot->slab_name);
Arnaldo Carvalho de Melo2e6599c2005-06-18 22:46:52 -07002832 prot->rsk_prot->slab = NULL;
2833 }
2834
Arnaldo Carvalho de Melo6d6ee432005-12-13 23:25:19 -08002835 if (prot->twsk_prot != NULL && prot->twsk_prot->twsk_slab != NULL) {
Arnaldo Carvalho de Melo6d6ee432005-12-13 23:25:19 -08002836 kmem_cache_destroy(prot->twsk_prot->twsk_slab);
Catalin Marinas7e56b5d2008-11-21 16:45:22 -08002837 kfree(prot->twsk_prot->twsk_slab_name);
Arnaldo Carvalho de Melo6d6ee432005-12-13 23:25:19 -08002838 prot->twsk_prot->twsk_slab = NULL;
Arnaldo Carvalho de Melo8feaf0c02005-08-09 20:09:30 -07002839 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002840}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002841EXPORT_SYMBOL(proto_unregister);
2842
2843#ifdef CONFIG_PROC_FS
Linus Torvalds1da177e2005-04-16 15:20:36 -07002844static void *proto_seq_start(struct seq_file *seq, loff_t *pos)
Glauber Costa36b77a52011-12-16 00:51:59 +00002845 __acquires(proto_list_mutex)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002846{
Glauber Costa36b77a52011-12-16 00:51:59 +00002847 mutex_lock(&proto_list_mutex);
Pavel Emelianov60f04382007-07-09 13:15:14 -07002848 return seq_list_start_head(&proto_list, *pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002849}
2850
2851static void *proto_seq_next(struct seq_file *seq, void *v, loff_t *pos)
2852{
Pavel Emelianov60f04382007-07-09 13:15:14 -07002853 return seq_list_next(v, &proto_list, pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002854}
2855
2856static void proto_seq_stop(struct seq_file *seq, void *v)
Glauber Costa36b77a52011-12-16 00:51:59 +00002857 __releases(proto_list_mutex)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002858{
Glauber Costa36b77a52011-12-16 00:51:59 +00002859 mutex_unlock(&proto_list_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002860}
2861
2862static char proto_method_implemented(const void *method)
2863{
2864 return method == NULL ? 'n' : 'y';
2865}
Glauber Costa180d8cd2011-12-11 21:47:02 +00002866static long sock_prot_memory_allocated(struct proto *proto)
2867{
Jeffrin Josecb75a362012-04-25 19:17:29 +05302868 return proto->memory_allocated != NULL ? proto_memory_allocated(proto) : -1L;
Glauber Costa180d8cd2011-12-11 21:47:02 +00002869}
2870
2871static char *sock_prot_memory_pressure(struct proto *proto)
2872{
2873 return proto->memory_pressure != NULL ?
2874 proto_memory_pressure(proto) ? "yes" : "no" : "NI";
2875}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002876
2877static void proto_seq_printf(struct seq_file *seq, struct proto *proto)
2878{
Glauber Costa180d8cd2011-12-11 21:47:02 +00002879
Eric Dumazet8d987e52010-11-09 23:24:26 +00002880 seq_printf(seq, "%-9s %4u %6d %6ld %-3s %6u %-3s %-10s "
Linus Torvalds1da177e2005-04-16 15:20:36 -07002881 "%2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c\n",
2882 proto->name,
2883 proto->obj_size,
Eric Dumazet14e943d2008-11-19 15:14:01 -08002884 sock_prot_inuse_get(seq_file_net(seq), proto),
Glauber Costa180d8cd2011-12-11 21:47:02 +00002885 sock_prot_memory_allocated(proto),
2886 sock_prot_memory_pressure(proto),
Linus Torvalds1da177e2005-04-16 15:20:36 -07002887 proto->max_header,
2888 proto->slab == NULL ? "no" : "yes",
2889 module_name(proto->owner),
2890 proto_method_implemented(proto->close),
2891 proto_method_implemented(proto->connect),
2892 proto_method_implemented(proto->disconnect),
2893 proto_method_implemented(proto->accept),
2894 proto_method_implemented(proto->ioctl),
2895 proto_method_implemented(proto->init),
2896 proto_method_implemented(proto->destroy),
2897 proto_method_implemented(proto->shutdown),
2898 proto_method_implemented(proto->setsockopt),
2899 proto_method_implemented(proto->getsockopt),
2900 proto_method_implemented(proto->sendmsg),
2901 proto_method_implemented(proto->recvmsg),
2902 proto_method_implemented(proto->sendpage),
2903 proto_method_implemented(proto->bind),
2904 proto_method_implemented(proto->backlog_rcv),
2905 proto_method_implemented(proto->hash),
2906 proto_method_implemented(proto->unhash),
2907 proto_method_implemented(proto->get_port),
2908 proto_method_implemented(proto->enter_memory_pressure));
2909}
2910
2911static int proto_seq_show(struct seq_file *seq, void *v)
2912{
Pavel Emelianov60f04382007-07-09 13:15:14 -07002913 if (v == &proto_list)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002914 seq_printf(seq, "%-9s %-4s %-8s %-6s %-5s %-7s %-4s %-10s %s",
2915 "protocol",
2916 "size",
2917 "sockets",
2918 "memory",
2919 "press",
2920 "maxhdr",
2921 "slab",
2922 "module",
2923 "cl co di ac io in de sh ss gs se re sp bi br ha uh gp em\n");
2924 else
Pavel Emelianov60f04382007-07-09 13:15:14 -07002925 proto_seq_printf(seq, list_entry(v, struct proto, node));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002926 return 0;
2927}
2928
Stephen Hemmingerf6908082007-03-12 14:34:29 -07002929static const struct seq_operations proto_seq_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002930 .start = proto_seq_start,
2931 .next = proto_seq_next,
2932 .stop = proto_seq_stop,
2933 .show = proto_seq_show,
2934};
2935
2936static int proto_seq_open(struct inode *inode, struct file *file)
2937{
Eric Dumazet14e943d2008-11-19 15:14:01 -08002938 return seq_open_net(inode, file, &proto_seq_ops,
2939 sizeof(struct seq_net_private));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002940}
2941
Arjan van de Ven9a321442007-02-12 00:55:35 -08002942static const struct file_operations proto_seq_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002943 .owner = THIS_MODULE,
2944 .open = proto_seq_open,
2945 .read = seq_read,
2946 .llseek = seq_lseek,
Eric Dumazet14e943d2008-11-19 15:14:01 -08002947 .release = seq_release_net,
2948};
2949
2950static __net_init int proto_init_net(struct net *net)
2951{
Gao fengd4beaa62013-02-18 01:34:54 +00002952 if (!proc_create("protocols", S_IRUGO, net->proc_net, &proto_seq_fops))
Eric Dumazet14e943d2008-11-19 15:14:01 -08002953 return -ENOMEM;
2954
2955 return 0;
2956}
2957
2958static __net_exit void proto_exit_net(struct net *net)
2959{
Gao fengece31ff2013-02-18 01:34:56 +00002960 remove_proc_entry("protocols", net->proc_net);
Eric Dumazet14e943d2008-11-19 15:14:01 -08002961}
2962
2963
2964static __net_initdata struct pernet_operations proto_net_ops = {
2965 .init = proto_init_net,
2966 .exit = proto_exit_net,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002967};
2968
2969static int __init proto_init(void)
2970{
Eric Dumazet14e943d2008-11-19 15:14:01 -08002971 return register_pernet_subsys(&proto_net_ops);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002972}
2973
2974subsys_initcall(proto_init);
2975
2976#endif /* PROC_FS */