blob: ad8a4bc841267a442a1da3c56ef1cf074f9825b9 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * Generic socket support routines. Memory allocators, socket lock/release
7 * handler for protocols to use and generic option handler.
8 *
9 *
Jesper Juhl02c30a82005-05-05 16:16:16 -070010 * Authors: Ross Biro
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Florian La Roche, <flla@stud.uni-sb.de>
13 * Alan Cox, <A.Cox@swansea.ac.uk>
14 *
15 * Fixes:
16 * Alan Cox : Numerous verify_area() problems
17 * Alan Cox : Connecting on a connecting socket
18 * now returns an error for tcp.
19 * Alan Cox : sock->protocol is set correctly.
20 * and is not sometimes left as 0.
21 * Alan Cox : connect handles icmp errors on a
22 * connect properly. Unfortunately there
23 * is a restart syscall nasty there. I
24 * can't match BSD without hacking the C
25 * library. Ideas urgently sought!
26 * Alan Cox : Disallow bind() to addresses that are
27 * not ours - especially broadcast ones!!
28 * Alan Cox : Socket 1024 _IS_ ok for users. (fencepost)
29 * Alan Cox : sock_wfree/sock_rfree don't destroy sockets,
30 * instead they leave that for the DESTROY timer.
31 * Alan Cox : Clean up error flag in accept
32 * Alan Cox : TCP ack handling is buggy, the DESTROY timer
33 * was buggy. Put a remove_sock() in the handler
34 * for memory when we hit 0. Also altered the timer
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +090035 * code. The ACK stuff can wait and needs major
Linus Torvalds1da177e2005-04-16 15:20:36 -070036 * TCP layer surgery.
37 * Alan Cox : Fixed TCP ack bug, removed remove sock
38 * and fixed timer/inet_bh race.
39 * Alan Cox : Added zapped flag for TCP
40 * Alan Cox : Move kfree_skb into skbuff.c and tidied up surplus code
41 * Alan Cox : for new sk_buff allocations wmalloc/rmalloc now call alloc_skb
42 * Alan Cox : kfree_s calls now are kfree_skbmem so we can track skb resources
43 * Alan Cox : Supports socket option broadcast now as does udp. Packet and raw need fixing.
44 * Alan Cox : Added RCVBUF,SNDBUF size setting. It suddenly occurred to me how easy it was so...
45 * Rick Sladkey : Relaxed UDP rules for matching packets.
46 * C.E.Hawkins : IFF_PROMISC/SIOCGHWADDR support
47 * Pauline Middelink : identd support
48 * Alan Cox : Fixed connect() taking signals I think.
49 * Alan Cox : SO_LINGER supported
50 * Alan Cox : Error reporting fixes
51 * Anonymous : inet_create tidied up (sk->reuse setting)
52 * Alan Cox : inet sockets don't set sk->type!
53 * Alan Cox : Split socket option code
54 * Alan Cox : Callbacks
55 * Alan Cox : Nagle flag for Charles & Johannes stuff
56 * Alex : Removed restriction on inet fioctl
57 * Alan Cox : Splitting INET from NET core
58 * Alan Cox : Fixed bogus SO_TYPE handling in getsockopt()
59 * Adam Caldwell : Missing return in SO_DONTROUTE/SO_DEBUG code
60 * Alan Cox : Split IP from generic code
61 * Alan Cox : New kfree_skbmem()
62 * Alan Cox : Make SO_DEBUG superuser only.
63 * Alan Cox : Allow anyone to clear SO_DEBUG
64 * (compatibility fix)
65 * Alan Cox : Added optimistic memory grabbing for AF_UNIX throughput.
66 * Alan Cox : Allocator for a socket is settable.
67 * Alan Cox : SO_ERROR includes soft errors.
68 * Alan Cox : Allow NULL arguments on some SO_ opts
69 * Alan Cox : Generic socket allocation to make hooks
70 * easier (suggested by Craig Metz).
71 * Michael Pall : SO_ERROR returns positive errno again
72 * Steve Whitehouse: Added default destructor to free
73 * protocol private data.
74 * Steve Whitehouse: Added various other default routines
75 * common to several socket families.
76 * Chris Evans : Call suser() check last on F_SETOWN
77 * Jay Schulist : Added SO_ATTACH_FILTER and SO_DETACH_FILTER.
78 * Andi Kleen : Add sock_kmalloc()/sock_kfree_s()
79 * Andi Kleen : Fix write_space callback
80 * Chris Evans : Security fixes - signedness again
81 * Arnaldo C. Melo : cleanups, use skb_queue_purge
82 *
83 * To Fix:
84 *
85 *
86 * This program is free software; you can redistribute it and/or
87 * modify it under the terms of the GNU General Public License
88 * as published by the Free Software Foundation; either version
89 * 2 of the License, or (at your option) any later version.
90 */
91
Joe Perchese005d192012-05-16 19:58:40 +000092#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
93
Randy Dunlap4fc268d2006-01-11 12:17:47 -080094#include <linux/capability.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070095#include <linux/errno.h>
Richard Cochrancb820f82013-07-19 19:40:09 +020096#include <linux/errqueue.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070097#include <linux/types.h>
98#include <linux/socket.h>
99#include <linux/in.h>
100#include <linux/kernel.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101#include <linux/module.h>
102#include <linux/proc_fs.h>
103#include <linux/seq_file.h>
104#include <linux/sched.h>
Vlastimil Babkaf1083042017-05-08 15:59:53 -0700105#include <linux/sched/mm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106#include <linux/timer.h>
107#include <linux/string.h>
108#include <linux/sockios.h>
109#include <linux/net.h>
110#include <linux/mm.h>
111#include <linux/slab.h>
112#include <linux/interrupt.h>
113#include <linux/poll.h>
114#include <linux/tcp.h>
115#include <linux/init.h>
Al Viroa1f8e7f72006-10-19 16:08:53 -0400116#include <linux/highmem.h>
Eric W. Biederman3f551f92010-06-13 03:28:59 +0000117#include <linux/user_namespace.h>
Ingo Molnarc5905af2012-02-24 08:31:31 +0100118#include <linux/static_key.h>
David S. Miller3969eb32012-01-09 13:44:23 -0800119#include <linux/memcontrol.h>
David S. Miller8c1ae102012-05-03 02:25:55 -0400120#include <linux/prefetch.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700121
Linus Torvalds7c0f6ba2016-12-24 11:46:01 -0800122#include <linux/uaccess.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700123
124#include <linux/netdevice.h>
125#include <net/protocol.h>
126#include <linux/skbuff.h>
Eric W. Biederman457c4cb2007-09-12 12:01:34 +0200127#include <net/net_namespace.h>
Arnaldo Carvalho de Melo2e6599c2005-06-18 22:46:52 -0700128#include <net/request_sock.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700129#include <net/sock.h>
Patrick Ohly20d49472009-02-12 05:03:38 +0000130#include <linux/net_tstamp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700131#include <net/xfrm.h>
132#include <linux/ipsec.h>
Herbert Xuf8451722010-05-24 00:12:34 -0700133#include <net/cls_cgroup.h>
Neil Horman5bc14212011-11-22 05:10:51 +0000134#include <net/netprio_cgroup.h>
Craig Gallekeb4cb002015-06-15 11:26:18 -0400135#include <linux/sock_diag.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700136
137#include <linux/filter.h>
Craig Gallek538950a2016-01-04 17:41:47 -0500138#include <net/sock_reuseport.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700139
Satoru Moriya3847ce32011-06-17 12:00:03 +0000140#include <trace/events/sock.h>
141
Linus Torvalds1da177e2005-04-16 15:20:36 -0700142#include <net/tcp.h>
Eliezer Tamir076bb0c2013-07-10 17:13:17 +0300143#include <net/busy_poll.h>
Eliezer Tamir06021292013-06-10 11:39:50 +0300144
Glauber Costa36b77a52011-12-16 00:51:59 +0000145static DEFINE_MUTEX(proto_list_mutex);
Glauber Costad1a4c0b2011-12-11 21:47:04 +0000146static LIST_HEAD(proto_list);
147
Eric W. Biedermana3b299d2014-04-23 14:26:56 -0700148/**
149 * sk_ns_capable - General socket capability test
150 * @sk: Socket to use a capability on or through
151 * @user_ns: The user namespace of the capability to use
152 * @cap: The capability to use
153 *
154 * Test to see if the opener of the socket had when the socket was
155 * created and the current process has the capability @cap in the user
156 * namespace @user_ns.
157 */
158bool sk_ns_capable(const struct sock *sk,
159 struct user_namespace *user_ns, int cap)
160{
161 return file_ns_capable(sk->sk_socket->file, user_ns, cap) &&
162 ns_capable(user_ns, cap);
163}
164EXPORT_SYMBOL(sk_ns_capable);
165
166/**
167 * sk_capable - Socket global capability test
168 * @sk: Socket to use a capability on or through
Masanari Iidae793c0f2014-09-04 23:44:36 +0900169 * @cap: The global capability to use
Eric W. Biedermana3b299d2014-04-23 14:26:56 -0700170 *
171 * Test to see if the opener of the socket had when the socket was
172 * created and the current process has the capability @cap in all user
173 * namespaces.
174 */
175bool sk_capable(const struct sock *sk, int cap)
176{
177 return sk_ns_capable(sk, &init_user_ns, cap);
178}
179EXPORT_SYMBOL(sk_capable);
180
181/**
182 * sk_net_capable - Network namespace socket capability test
183 * @sk: Socket to use a capability on or through
184 * @cap: The capability to use
185 *
Masanari Iidae793c0f2014-09-04 23:44:36 +0900186 * Test to see if the opener of the socket had when the socket was created
Eric W. Biedermana3b299d2014-04-23 14:26:56 -0700187 * and the current process has the capability @cap over the network namespace
188 * the socket is a member of.
189 */
190bool sk_net_capable(const struct sock *sk, int cap)
191{
192 return sk_ns_capable(sk, sock_net(sk)->user_ns, cap);
193}
194EXPORT_SYMBOL(sk_net_capable);
195
Ingo Molnarda21f242006-07-03 00:25:12 -0700196/*
197 * Each address family might have different locking rules, so we have
David Howellscdfbabf2017-03-09 08:09:05 +0000198 * one slock key per address family and separate keys for internal and
199 * userspace sockets.
Ingo Molnarda21f242006-07-03 00:25:12 -0700200 */
Ingo Molnara5b5bb92006-07-03 00:25:35 -0700201static struct lock_class_key af_family_keys[AF_MAX];
David Howellscdfbabf2017-03-09 08:09:05 +0000202static struct lock_class_key af_family_kern_keys[AF_MAX];
Ingo Molnara5b5bb92006-07-03 00:25:35 -0700203static struct lock_class_key af_family_slock_keys[AF_MAX];
David Howellscdfbabf2017-03-09 08:09:05 +0000204static struct lock_class_key af_family_kern_slock_keys[AF_MAX];
Ingo Molnara5b5bb92006-07-03 00:25:35 -0700205
Ingo Molnara5b5bb92006-07-03 00:25:35 -0700206/*
207 * Make lock validator output more readable. (we pre-construct these
208 * strings build-time, so that runtime initialization of socket
209 * locks is fast):
210 */
David Howellscdfbabf2017-03-09 08:09:05 +0000211
212#define _sock_locks(x) \
213 x "AF_UNSPEC", x "AF_UNIX" , x "AF_INET" , \
214 x "AF_AX25" , x "AF_IPX" , x "AF_APPLETALK", \
215 x "AF_NETROM", x "AF_BRIDGE" , x "AF_ATMPVC" , \
216 x "AF_X25" , x "AF_INET6" , x "AF_ROSE" , \
217 x "AF_DECnet", x "AF_NETBEUI" , x "AF_SECURITY" , \
218 x "AF_KEY" , x "AF_NETLINK" , x "AF_PACKET" , \
219 x "AF_ASH" , x "AF_ECONET" , x "AF_ATMSVC" , \
220 x "AF_RDS" , x "AF_SNA" , x "AF_IRDA" , \
221 x "AF_PPPOX" , x "AF_WANPIPE" , x "AF_LLC" , \
222 x "27" , x "28" , x "AF_CAN" , \
223 x "AF_TIPC" , x "AF_BLUETOOTH", x "IUCV" , \
224 x "AF_RXRPC" , x "AF_ISDN" , x "AF_PHONET" , \
225 x "AF_IEEE802154", x "AF_CAIF" , x "AF_ALG" , \
226 x "AF_NFC" , x "AF_VSOCK" , x "AF_KCM" , \
227 x "AF_QIPCRTR", x "AF_SMC" , x "AF_MAX"
228
Jan Engelhardt36cbd3d2009-08-05 10:42:58 -0700229static const char *const af_family_key_strings[AF_MAX+1] = {
David Howellscdfbabf2017-03-09 08:09:05 +0000230 _sock_locks("sk_lock-")
Ingo Molnara5b5bb92006-07-03 00:25:35 -0700231};
Jan Engelhardt36cbd3d2009-08-05 10:42:58 -0700232static const char *const af_family_slock_key_strings[AF_MAX+1] = {
David Howellscdfbabf2017-03-09 08:09:05 +0000233 _sock_locks("slock-")
Ingo Molnara5b5bb92006-07-03 00:25:35 -0700234};
Jan Engelhardt36cbd3d2009-08-05 10:42:58 -0700235static const char *const af_family_clock_key_strings[AF_MAX+1] = {
David Howellscdfbabf2017-03-09 08:09:05 +0000236 _sock_locks("clock-")
237};
238
239static const char *const af_family_kern_key_strings[AF_MAX+1] = {
240 _sock_locks("k-sk_lock-")
241};
242static const char *const af_family_kern_slock_key_strings[AF_MAX+1] = {
243 _sock_locks("k-slock-")
244};
245static const char *const af_family_kern_clock_key_strings[AF_MAX+1] = {
246 _sock_locks("k-clock-")
Peter Zijlstra443aef0e2007-07-19 01:49:00 -0700247};
Paolo Abeni581319c2017-03-09 13:54:08 +0100248static const char *const af_family_rlock_key_strings[AF_MAX+1] = {
249 "rlock-AF_UNSPEC", "rlock-AF_UNIX" , "rlock-AF_INET" ,
250 "rlock-AF_AX25" , "rlock-AF_IPX" , "rlock-AF_APPLETALK",
251 "rlock-AF_NETROM", "rlock-AF_BRIDGE" , "rlock-AF_ATMPVC" ,
252 "rlock-AF_X25" , "rlock-AF_INET6" , "rlock-AF_ROSE" ,
253 "rlock-AF_DECnet", "rlock-AF_NETBEUI" , "rlock-AF_SECURITY" ,
254 "rlock-AF_KEY" , "rlock-AF_NETLINK" , "rlock-AF_PACKET" ,
255 "rlock-AF_ASH" , "rlock-AF_ECONET" , "rlock-AF_ATMSVC" ,
256 "rlock-AF_RDS" , "rlock-AF_SNA" , "rlock-AF_IRDA" ,
257 "rlock-AF_PPPOX" , "rlock-AF_WANPIPE" , "rlock-AF_LLC" ,
258 "rlock-27" , "rlock-28" , "rlock-AF_CAN" ,
259 "rlock-AF_TIPC" , "rlock-AF_BLUETOOTH", "rlock-AF_IUCV" ,
260 "rlock-AF_RXRPC" , "rlock-AF_ISDN" , "rlock-AF_PHONET" ,
261 "rlock-AF_IEEE802154", "rlock-AF_CAIF" , "rlock-AF_ALG" ,
262 "rlock-AF_NFC" , "rlock-AF_VSOCK" , "rlock-AF_KCM" ,
263 "rlock-AF_QIPCRTR", "rlock-AF_SMC" , "rlock-AF_MAX"
264};
265static const char *const af_family_wlock_key_strings[AF_MAX+1] = {
266 "wlock-AF_UNSPEC", "wlock-AF_UNIX" , "wlock-AF_INET" ,
267 "wlock-AF_AX25" , "wlock-AF_IPX" , "wlock-AF_APPLETALK",
268 "wlock-AF_NETROM", "wlock-AF_BRIDGE" , "wlock-AF_ATMPVC" ,
269 "wlock-AF_X25" , "wlock-AF_INET6" , "wlock-AF_ROSE" ,
270 "wlock-AF_DECnet", "wlock-AF_NETBEUI" , "wlock-AF_SECURITY" ,
271 "wlock-AF_KEY" , "wlock-AF_NETLINK" , "wlock-AF_PACKET" ,
272 "wlock-AF_ASH" , "wlock-AF_ECONET" , "wlock-AF_ATMSVC" ,
273 "wlock-AF_RDS" , "wlock-AF_SNA" , "wlock-AF_IRDA" ,
274 "wlock-AF_PPPOX" , "wlock-AF_WANPIPE" , "wlock-AF_LLC" ,
275 "wlock-27" , "wlock-28" , "wlock-AF_CAN" ,
276 "wlock-AF_TIPC" , "wlock-AF_BLUETOOTH", "wlock-AF_IUCV" ,
277 "wlock-AF_RXRPC" , "wlock-AF_ISDN" , "wlock-AF_PHONET" ,
278 "wlock-AF_IEEE802154", "wlock-AF_CAIF" , "wlock-AF_ALG" ,
279 "wlock-AF_NFC" , "wlock-AF_VSOCK" , "wlock-AF_KCM" ,
280 "wlock-AF_QIPCRTR", "wlock-AF_SMC" , "wlock-AF_MAX"
281};
282static const char *const af_family_elock_key_strings[AF_MAX+1] = {
283 "elock-AF_UNSPEC", "elock-AF_UNIX" , "elock-AF_INET" ,
284 "elock-AF_AX25" , "elock-AF_IPX" , "elock-AF_APPLETALK",
285 "elock-AF_NETROM", "elock-AF_BRIDGE" , "elock-AF_ATMPVC" ,
286 "elock-AF_X25" , "elock-AF_INET6" , "elock-AF_ROSE" ,
287 "elock-AF_DECnet", "elock-AF_NETBEUI" , "elock-AF_SECURITY" ,
288 "elock-AF_KEY" , "elock-AF_NETLINK" , "elock-AF_PACKET" ,
289 "elock-AF_ASH" , "elock-AF_ECONET" , "elock-AF_ATMSVC" ,
290 "elock-AF_RDS" , "elock-AF_SNA" , "elock-AF_IRDA" ,
291 "elock-AF_PPPOX" , "elock-AF_WANPIPE" , "elock-AF_LLC" ,
292 "elock-27" , "elock-28" , "elock-AF_CAN" ,
293 "elock-AF_TIPC" , "elock-AF_BLUETOOTH", "elock-AF_IUCV" ,
294 "elock-AF_RXRPC" , "elock-AF_ISDN" , "elock-AF_PHONET" ,
295 "elock-AF_IEEE802154", "elock-AF_CAIF" , "elock-AF_ALG" ,
296 "elock-AF_NFC" , "elock-AF_VSOCK" , "elock-AF_KCM" ,
297 "elock-AF_QIPCRTR", "elock-AF_SMC" , "elock-AF_MAX"
298};
Ingo Molnarda21f242006-07-03 00:25:12 -0700299
300/*
Paolo Abeni581319c2017-03-09 13:54:08 +0100301 * sk_callback_lock and sk queues locking rules are per-address-family,
Ingo Molnarda21f242006-07-03 00:25:12 -0700302 * so split the lock classes by using a per-AF key:
303 */
304static struct lock_class_key af_callback_keys[AF_MAX];
Paolo Abeni581319c2017-03-09 13:54:08 +0100305static struct lock_class_key af_rlock_keys[AF_MAX];
306static struct lock_class_key af_wlock_keys[AF_MAX];
307static struct lock_class_key af_elock_keys[AF_MAX];
David Howellscdfbabf2017-03-09 08:09:05 +0000308static struct lock_class_key af_kern_callback_keys[AF_MAX];
Ingo Molnarda21f242006-07-03 00:25:12 -0700309
Linus Torvalds1da177e2005-04-16 15:20:36 -0700310/* Take into consideration the size of the struct sk_buff overhead in the
311 * determination of these values, since that is non-constant across
312 * platforms. This makes socket queueing behavior and performance
313 * not depend upon such differences.
314 */
315#define _SK_MEM_PACKETS 256
Eric Dumazet87fb4b72011-10-13 07:28:54 +0000316#define _SK_MEM_OVERHEAD SKB_TRUESIZE(256)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700317#define SK_WMEM_MAX (_SK_MEM_OVERHEAD * _SK_MEM_PACKETS)
318#define SK_RMEM_MAX (_SK_MEM_OVERHEAD * _SK_MEM_PACKETS)
319
320/* Run time adjustable parameters. */
Brian Haleyab32ea52006-09-22 14:15:41 -0700321__u32 sysctl_wmem_max __read_mostly = SK_WMEM_MAX;
Hans Schillstrom6d8ebc82012-04-30 08:13:50 +0200322EXPORT_SYMBOL(sysctl_wmem_max);
Brian Haleyab32ea52006-09-22 14:15:41 -0700323__u32 sysctl_rmem_max __read_mostly = SK_RMEM_MAX;
Hans Schillstrom6d8ebc82012-04-30 08:13:50 +0200324EXPORT_SYMBOL(sysctl_rmem_max);
Brian Haleyab32ea52006-09-22 14:15:41 -0700325__u32 sysctl_wmem_default __read_mostly = SK_WMEM_MAX;
326__u32 sysctl_rmem_default __read_mostly = SK_RMEM_MAX;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700327
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300328/* Maximal space eaten by iovec or ancillary data plus some space */
Brian Haleyab32ea52006-09-22 14:15:41 -0700329int sysctl_optmem_max __read_mostly = sizeof(unsigned long)*(2*UIO_MAXIOV+512);
Eric Dumazet2a915252009-05-27 11:30:05 +0000330EXPORT_SYMBOL(sysctl_optmem_max);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700331
Willem de Bruijnb245be12015-01-30 13:29:32 -0500332int sysctl_tstamp_allow_data __read_mostly = 1;
333
Mel Gormanc93bdd02012-07-31 16:44:19 -0700334struct static_key memalloc_socks = STATIC_KEY_INIT_FALSE;
335EXPORT_SYMBOL_GPL(memalloc_socks);
336
Mel Gorman7cb02402012-07-31 16:44:16 -0700337/**
338 * sk_set_memalloc - sets %SOCK_MEMALLOC
339 * @sk: socket to set it on
340 *
341 * Set %SOCK_MEMALLOC on a socket for access to emergency reserves.
342 * It's the responsibility of the admin to adjust min_free_kbytes
343 * to meet the requirements
344 */
345void sk_set_memalloc(struct sock *sk)
346{
347 sock_set_flag(sk, SOCK_MEMALLOC);
348 sk->sk_allocation |= __GFP_MEMALLOC;
Mel Gormanc93bdd02012-07-31 16:44:19 -0700349 static_key_slow_inc(&memalloc_socks);
Mel Gorman7cb02402012-07-31 16:44:16 -0700350}
351EXPORT_SYMBOL_GPL(sk_set_memalloc);
352
353void sk_clear_memalloc(struct sock *sk)
354{
355 sock_reset_flag(sk, SOCK_MEMALLOC);
356 sk->sk_allocation &= ~__GFP_MEMALLOC;
Mel Gormanc93bdd02012-07-31 16:44:19 -0700357 static_key_slow_dec(&memalloc_socks);
Mel Gormanc76562b2012-07-31 16:44:41 -0700358
359 /*
360 * SOCK_MEMALLOC is allowed to ignore rmem limits to ensure forward
Mel Gorman5d753612015-06-10 21:02:04 -0400361 * progress of swapping. SOCK_MEMALLOC may be cleared while
362 * it has rmem allocations due to the last swapfile being deactivated
363 * but there is a risk that the socket is unusable due to exceeding
364 * the rmem limits. Reclaim the reserves and obey rmem limits again.
Mel Gormanc76562b2012-07-31 16:44:41 -0700365 */
Mel Gorman5d753612015-06-10 21:02:04 -0400366 sk_mem_reclaim(sk);
Mel Gorman7cb02402012-07-31 16:44:16 -0700367}
368EXPORT_SYMBOL_GPL(sk_clear_memalloc);
369
Mel Gormanb4b9e352012-07-31 16:44:26 -0700370int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb)
371{
372 int ret;
Vlastimil Babkaf1083042017-05-08 15:59:53 -0700373 unsigned int noreclaim_flag;
Mel Gormanb4b9e352012-07-31 16:44:26 -0700374
375 /* these should have been dropped before queueing */
376 BUG_ON(!sock_flag(sk, SOCK_MEMALLOC));
377
Vlastimil Babkaf1083042017-05-08 15:59:53 -0700378 noreclaim_flag = memalloc_noreclaim_save();
Mel Gormanb4b9e352012-07-31 16:44:26 -0700379 ret = sk->sk_backlog_rcv(sk, skb);
Vlastimil Babkaf1083042017-05-08 15:59:53 -0700380 memalloc_noreclaim_restore(noreclaim_flag);
Mel Gormanb4b9e352012-07-31 16:44:26 -0700381
382 return ret;
383}
384EXPORT_SYMBOL(__sk_backlog_rcv);
385
Linus Torvalds1da177e2005-04-16 15:20:36 -0700386static int sock_set_timeout(long *timeo_p, char __user *optval, int optlen)
387{
388 struct timeval tv;
389
390 if (optlen < sizeof(tv))
391 return -EINVAL;
392 if (copy_from_user(&tv, optval, sizeof(tv)))
393 return -EFAULT;
Vasily Averinba780732007-05-24 16:58:54 -0700394 if (tv.tv_usec < 0 || tv.tv_usec >= USEC_PER_SEC)
395 return -EDOM;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700396
Vasily Averinba780732007-05-24 16:58:54 -0700397 if (tv.tv_sec < 0) {
Andrew Morton6f11df82007-07-09 13:16:00 -0700398 static int warned __read_mostly;
399
Vasily Averinba780732007-05-24 16:58:54 -0700400 *timeo_p = 0;
Ilpo Järvinen50aab542008-05-02 16:20:10 -0700401 if (warned < 10 && net_ratelimit()) {
Vasily Averinba780732007-05-24 16:58:54 -0700402 warned++;
Joe Perchese005d192012-05-16 19:58:40 +0000403 pr_info("%s: `%s' (pid %d) tries to set negative timeout\n",
404 __func__, current->comm, task_pid_nr(current));
Ilpo Järvinen50aab542008-05-02 16:20:10 -0700405 }
Vasily Averinba780732007-05-24 16:58:54 -0700406 return 0;
407 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700408 *timeo_p = MAX_SCHEDULE_TIMEOUT;
409 if (tv.tv_sec == 0 && tv.tv_usec == 0)
410 return 0;
411 if (tv.tv_sec < (MAX_SCHEDULE_TIMEOUT/HZ - 1))
Gao Feng8ccde4c2017-02-21 17:09:19 +0800412 *timeo_p = tv.tv_sec * HZ + DIV_ROUND_UP(tv.tv_usec, USEC_PER_SEC / HZ);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700413 return 0;
414}
415
416static void sock_warn_obsolete_bsdism(const char *name)
417{
418 static int warned;
419 static char warncomm[TASK_COMM_LEN];
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900420 if (strcmp(warncomm, current->comm) && warned < 5) {
421 strcpy(warncomm, current->comm);
Joe Perchese005d192012-05-16 19:58:40 +0000422 pr_warn("process `%s' is using obsolete %s SO_BSDCOMPAT\n",
423 warncomm, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700424 warned++;
425 }
426}
427
Hannes Frederic Sowa080a2702015-10-26 13:51:37 +0100428static bool sock_needs_netstamp(const struct sock *sk)
429{
430 switch (sk->sk_family) {
431 case AF_UNSPEC:
432 case AF_UNIX:
433 return false;
434 default:
435 return true;
436 }
437}
438
Eric Dumazet08e29af2011-11-28 12:04:18 +0000439static void sock_disable_timestamp(struct sock *sk, unsigned long flags)
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900440{
Eric Dumazet08e29af2011-11-28 12:04:18 +0000441 if (sk->sk_flags & flags) {
442 sk->sk_flags &= ~flags;
Hannes Frederic Sowa080a2702015-10-26 13:51:37 +0100443 if (sock_needs_netstamp(sk) &&
444 !(sk->sk_flags & SK_FLAGS_TIMESTAMP))
Patrick Ohly20d49472009-02-12 05:03:38 +0000445 net_disable_timestamp();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700446 }
447}
448
449
samanthakumare6afc8a2016-04-05 12:41:15 -0400450int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
Denis Vlasenkof0088a52006-03-28 01:08:21 -0800451{
Neil Horman3b885782009-10-12 13:26:31 -0700452 unsigned long flags;
453 struct sk_buff_head *list = &sk->sk_receive_queue;
Denis Vlasenkof0088a52006-03-28 01:08:21 -0800454
Eric Dumazet0fd7bac2011-12-21 07:11:44 +0000455 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf) {
Eric Dumazet766e90372009-10-14 20:40:11 -0700456 atomic_inc(&sk->sk_drops);
Satoru Moriya3847ce32011-06-17 12:00:03 +0000457 trace_sock_rcvqueue_full(sk, skb);
Eric Dumazet766e90372009-10-14 20:40:11 -0700458 return -ENOMEM;
Denis Vlasenkof0088a52006-03-28 01:08:21 -0800459 }
460
Mel Gormanc76562b2012-07-31 16:44:41 -0700461 if (!sk_rmem_schedule(sk, skb, skb->truesize)) {
Eric Dumazet766e90372009-10-14 20:40:11 -0700462 atomic_inc(&sk->sk_drops);
463 return -ENOBUFS;
Hideo Aoki3ab224b2007-12-31 00:11:19 -0800464 }
465
Denis Vlasenkof0088a52006-03-28 01:08:21 -0800466 skb->dev = NULL;
467 skb_set_owner_r(skb, sk);
David S. Miller49ad9592008-12-17 22:11:38 -0800468
Eric Dumazet7fee2262010-05-11 23:19:48 +0000469 /* we escape from rcu protected region, make sure we dont leak
470 * a norefcounted dst
471 */
472 skb_dst_force(skb);
473
Neil Horman3b885782009-10-12 13:26:31 -0700474 spin_lock_irqsave(&list->lock, flags);
Eyal Birger3bc3b962015-03-01 14:58:30 +0200475 sock_skb_set_dropcount(sk, skb);
Neil Horman3b885782009-10-12 13:26:31 -0700476 __skb_queue_tail(list, skb);
477 spin_unlock_irqrestore(&list->lock, flags);
Denis Vlasenkof0088a52006-03-28 01:08:21 -0800478
479 if (!sock_flag(sk, SOCK_DEAD))
David S. Miller676d2362014-04-11 16:15:36 -0400480 sk->sk_data_ready(sk);
Eric Dumazet766e90372009-10-14 20:40:11 -0700481 return 0;
Denis Vlasenkof0088a52006-03-28 01:08:21 -0800482}
samanthakumare6afc8a2016-04-05 12:41:15 -0400483EXPORT_SYMBOL(__sock_queue_rcv_skb);
484
485int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
486{
487 int err;
488
489 err = sk_filter(sk, skb);
490 if (err)
491 return err;
492
493 return __sock_queue_rcv_skb(sk, skb);
494}
Denis Vlasenkof0088a52006-03-28 01:08:21 -0800495EXPORT_SYMBOL(sock_queue_rcv_skb);
496
Willem de Bruijn4f0c40d92016-07-12 18:18:57 -0400497int __sk_receive_skb(struct sock *sk, struct sk_buff *skb,
Eric Dumazetc3f24cf2016-11-02 17:14:41 -0700498 const int nested, unsigned int trim_cap, bool refcounted)
Denis Vlasenkof0088a52006-03-28 01:08:21 -0800499{
500 int rc = NET_RX_SUCCESS;
501
Willem de Bruijn4f0c40d92016-07-12 18:18:57 -0400502 if (sk_filter_trim_cap(sk, skb, trim_cap))
Denis Vlasenkof0088a52006-03-28 01:08:21 -0800503 goto discard_and_relse;
504
505 skb->dev = NULL;
506
Sorin Dumitru274f4822014-07-22 21:16:51 +0300507 if (sk_rcvqueues_full(sk, sk->sk_rcvbuf)) {
Eric Dumazetc3774112010-04-27 15:13:20 -0700508 atomic_inc(&sk->sk_drops);
509 goto discard_and_relse;
510 }
Arnaldo Carvalho de Melo58a5a7b2006-11-16 14:06:06 -0200511 if (nested)
512 bh_lock_sock_nested(sk);
513 else
514 bh_lock_sock(sk);
Ingo Molnara5b5bb92006-07-03 00:25:35 -0700515 if (!sock_owned_by_user(sk)) {
516 /*
517 * trylock + unlock semantics:
518 */
519 mutex_acquire(&sk->sk_lock.dep_map, 0, 1, _RET_IP_);
520
Peter Zijlstrac57943a2008-10-07 14:18:42 -0700521 rc = sk_backlog_rcv(sk, skb);
Ingo Molnara5b5bb92006-07-03 00:25:35 -0700522
523 mutex_release(&sk->sk_lock.dep_map, 1, _RET_IP_);
Eric Dumazetf545a382012-04-22 23:34:26 +0000524 } else if (sk_add_backlog(sk, skb, sk->sk_rcvbuf)) {
Zhu Yi8eae9392010-03-04 18:01:40 +0000525 bh_unlock_sock(sk);
526 atomic_inc(&sk->sk_drops);
527 goto discard_and_relse;
528 }
529
Denis Vlasenkof0088a52006-03-28 01:08:21 -0800530 bh_unlock_sock(sk);
531out:
Eric Dumazetc3f24cf2016-11-02 17:14:41 -0700532 if (refcounted)
533 sock_put(sk);
Denis Vlasenkof0088a52006-03-28 01:08:21 -0800534 return rc;
535discard_and_relse:
536 kfree_skb(skb);
537 goto out;
538}
Willem de Bruijn4f0c40d92016-07-12 18:18:57 -0400539EXPORT_SYMBOL(__sk_receive_skb);
Denis Vlasenkof0088a52006-03-28 01:08:21 -0800540
541struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie)
542{
Eric Dumazetb6c67122010-04-08 23:03:29 +0000543 struct dst_entry *dst = __sk_dst_get(sk);
Denis Vlasenkof0088a52006-03-28 01:08:21 -0800544
545 if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) {
Krishna Kumare022f0b2009-10-19 23:46:20 +0000546 sk_tx_queue_clear(sk);
Julian Anastasov9b8805a2017-02-06 23:14:11 +0200547 sk->sk_dst_pending_confirm = 0;
Stephen Hemmingera9b3cd72011-08-01 16:19:00 +0000548 RCU_INIT_POINTER(sk->sk_dst_cache, NULL);
Denis Vlasenkof0088a52006-03-28 01:08:21 -0800549 dst_release(dst);
550 return NULL;
551 }
552
553 return dst;
554}
555EXPORT_SYMBOL(__sk_dst_check);
556
557struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie)
558{
559 struct dst_entry *dst = sk_dst_get(sk);
560
561 if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) {
562 sk_dst_reset(sk);
563 dst_release(dst);
564 return NULL;
565 }
566
567 return dst;
568}
569EXPORT_SYMBOL(sk_dst_check);
570
Brian Haleyc91f6df2012-11-26 05:21:08 +0000571static int sock_setbindtodevice(struct sock *sk, char __user *optval,
572 int optlen)
David S. Miller48788092007-09-14 16:41:03 -0700573{
574 int ret = -ENOPROTOOPT;
575#ifdef CONFIG_NETDEVICES
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +0900576 struct net *net = sock_net(sk);
David S. Miller48788092007-09-14 16:41:03 -0700577 char devname[IFNAMSIZ];
578 int index;
579
580 /* Sorry... */
581 ret = -EPERM;
Eric W. Biederman5e1fccc2012-11-16 03:03:04 +0000582 if (!ns_capable(net->user_ns, CAP_NET_RAW))
David S. Miller48788092007-09-14 16:41:03 -0700583 goto out;
584
585 ret = -EINVAL;
586 if (optlen < 0)
587 goto out;
588
589 /* Bind this socket to a particular device like "eth0",
590 * as specified in the passed interface name. If the
591 * name is "" or the option length is zero the socket
592 * is not bound.
593 */
594 if (optlen > IFNAMSIZ - 1)
595 optlen = IFNAMSIZ - 1;
596 memset(devname, 0, sizeof(devname));
597
598 ret = -EFAULT;
599 if (copy_from_user(devname, optval, optlen))
600 goto out;
601
David S. Miller000ba2e2009-11-05 22:37:11 -0800602 index = 0;
603 if (devname[0] != '\0') {
Eric Dumazetbf8e56b2009-11-05 21:03:39 -0800604 struct net_device *dev;
David S. Miller48788092007-09-14 16:41:03 -0700605
Eric Dumazetbf8e56b2009-11-05 21:03:39 -0800606 rcu_read_lock();
607 dev = dev_get_by_name_rcu(net, devname);
608 if (dev)
609 index = dev->ifindex;
610 rcu_read_unlock();
David S. Miller48788092007-09-14 16:41:03 -0700611 ret = -ENODEV;
612 if (!dev)
613 goto out;
David S. Miller48788092007-09-14 16:41:03 -0700614 }
615
616 lock_sock(sk);
617 sk->sk_bound_dev_if = index;
618 sk_dst_reset(sk);
619 release_sock(sk);
620
621 ret = 0;
622
623out:
624#endif
625
626 return ret;
627}
628
Brian Haleyc91f6df2012-11-26 05:21:08 +0000629static int sock_getbindtodevice(struct sock *sk, char __user *optval,
630 int __user *optlen, int len)
631{
632 int ret = -ENOPROTOOPT;
633#ifdef CONFIG_NETDEVICES
634 struct net *net = sock_net(sk);
Brian Haleyc91f6df2012-11-26 05:21:08 +0000635 char devname[IFNAMSIZ];
Brian Haleyc91f6df2012-11-26 05:21:08 +0000636
637 if (sk->sk_bound_dev_if == 0) {
638 len = 0;
639 goto zero;
640 }
641
642 ret = -EINVAL;
643 if (len < IFNAMSIZ)
644 goto out;
645
Nicolas Schichan5dbe7c12013-06-26 17:23:42 +0200646 ret = netdev_get_name(net, devname, sk->sk_bound_dev_if);
647 if (ret)
Brian Haleyc91f6df2012-11-26 05:21:08 +0000648 goto out;
Brian Haleyc91f6df2012-11-26 05:21:08 +0000649
650 len = strlen(devname) + 1;
651
652 ret = -EFAULT;
653 if (copy_to_user(optval, devname, len))
654 goto out;
655
656zero:
657 ret = -EFAULT;
658 if (put_user(len, optlen))
659 goto out;
660
661 ret = 0;
662
663out:
664#endif
665
666 return ret;
667}
668
Pavel Emelyanovc0ef8772007-11-15 03:03:19 -0800669static inline void sock_valbool_flag(struct sock *sk, int bit, int valbool)
670{
671 if (valbool)
672 sock_set_flag(sk, bit);
673 else
674 sock_reset_flag(sk, bit);
675}
676
hannes@stressinduktion.orgf60e5992015-04-01 17:07:44 +0200677bool sk_mc_loop(struct sock *sk)
678{
679 if (dev_recursion_level())
680 return false;
681 if (!sk)
682 return true;
683 switch (sk->sk_family) {
684 case AF_INET:
685 return inet_sk(sk)->mc_loop;
686#if IS_ENABLED(CONFIG_IPV6)
687 case AF_INET6:
688 return inet6_sk(sk)->mc_loop;
689#endif
690 }
691 WARN_ON(1);
692 return true;
693}
694EXPORT_SYMBOL(sk_mc_loop);
695
Linus Torvalds1da177e2005-04-16 15:20:36 -0700696/*
697 * This is meant for all protocols to use and covers goings on
698 * at the socket level. Everything here is generic.
699 */
700
701int sock_setsockopt(struct socket *sock, int level, int optname,
David S. Millerb7058842009-09-30 16:12:20 -0700702 char __user *optval, unsigned int optlen)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700703{
Eric Dumazet2a915252009-05-27 11:30:05 +0000704 struct sock *sk = sock->sk;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700705 int val;
706 int valbool;
707 struct linger ling;
708 int ret = 0;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900709
Linus Torvalds1da177e2005-04-16 15:20:36 -0700710 /*
711 * Options without arguments
712 */
713
David S. Miller48788092007-09-14 16:41:03 -0700714 if (optname == SO_BINDTODEVICE)
Brian Haleyc91f6df2012-11-26 05:21:08 +0000715 return sock_setbindtodevice(sk, optval, optlen);
David S. Miller48788092007-09-14 16:41:03 -0700716
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700717 if (optlen < sizeof(int))
718 return -EINVAL;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900719
Linus Torvalds1da177e2005-04-16 15:20:36 -0700720 if (get_user(val, (int __user *)optval))
721 return -EFAULT;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900722
Eric Dumazet2a915252009-05-27 11:30:05 +0000723 valbool = val ? 1 : 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700724
725 lock_sock(sk);
726
Eric Dumazet2a915252009-05-27 11:30:05 +0000727 switch (optname) {
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700728 case SO_DEBUG:
Eric Dumazet2a915252009-05-27 11:30:05 +0000729 if (val && !capable(CAP_NET_ADMIN))
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700730 ret = -EACCES;
Eric Dumazet2a915252009-05-27 11:30:05 +0000731 else
Pavel Emelyanovc0ef8772007-11-15 03:03:19 -0800732 sock_valbool_flag(sk, SOCK_DBG, valbool);
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700733 break;
734 case SO_REUSEADDR:
Pavel Emelyanov4a17fd52012-04-19 03:39:36 +0000735 sk->sk_reuse = (valbool ? SK_CAN_REUSE : SK_NO_REUSE);
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700736 break;
Tom Herbert055dc212013-01-22 09:49:50 +0000737 case SO_REUSEPORT:
738 sk->sk_reuseport = valbool;
739 break;
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700740 case SO_TYPE:
Jan Engelhardt49c794e2009-08-04 07:28:28 +0000741 case SO_PROTOCOL:
Jan Engelhardt0d6038e2009-08-04 07:28:29 +0000742 case SO_DOMAIN:
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700743 case SO_ERROR:
744 ret = -ENOPROTOOPT;
745 break;
746 case SO_DONTROUTE:
Pavel Emelyanovc0ef8772007-11-15 03:03:19 -0800747 sock_valbool_flag(sk, SOCK_LOCALROUTE, valbool);
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700748 break;
749 case SO_BROADCAST:
750 sock_valbool_flag(sk, SOCK_BROADCAST, valbool);
751 break;
752 case SO_SNDBUF:
753 /* Don't error on this BSD doesn't and if you think
Eric Dumazet82981932012-04-26 20:07:59 +0000754 * about it this is right. Otherwise apps have to
755 * play 'guess the biggest size' games. RCVBUF/SNDBUF
756 * are treated in BSD as hints
757 */
758 val = min_t(u32, val, sysctl_wmem_max);
Patrick McHardyb0573de2005-08-09 19:30:51 -0700759set_sndbuf:
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700760 sk->sk_userlocks |= SOCK_SNDBUF_LOCK;
Eric Dumazetb98b0bc2016-12-02 09:44:53 -0800761 sk->sk_sndbuf = max_t(int, val * 2, SOCK_MIN_SNDBUF);
Eric Dumazet82981932012-04-26 20:07:59 +0000762 /* Wake up sending tasks if we upped the value. */
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700763 sk->sk_write_space(sk);
764 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700765
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700766 case SO_SNDBUFFORCE:
767 if (!capable(CAP_NET_ADMIN)) {
768 ret = -EPERM;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700769 break;
770 }
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700771 goto set_sndbuf;
772
773 case SO_RCVBUF:
774 /* Don't error on this BSD doesn't and if you think
Eric Dumazet82981932012-04-26 20:07:59 +0000775 * about it this is right. Otherwise apps have to
776 * play 'guess the biggest size' games. RCVBUF/SNDBUF
777 * are treated in BSD as hints
778 */
779 val = min_t(u32, val, sysctl_rmem_max);
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700780set_rcvbuf:
781 sk->sk_userlocks |= SOCK_RCVBUF_LOCK;
782 /*
783 * We double it on the way in to account for
784 * "struct sk_buff" etc. overhead. Applications
785 * assume that the SO_RCVBUF setting they make will
786 * allow that much actual data to be received on that
787 * socket.
788 *
789 * Applications are unaware that "struct sk_buff" and
790 * other overheads allocate from the receive buffer
791 * during socket buffer allocation.
792 *
793 * And after considering the possible alternatives,
794 * returning the value we actually used in getsockopt
795 * is the most desirable behavior.
796 */
Eric Dumazetb98b0bc2016-12-02 09:44:53 -0800797 sk->sk_rcvbuf = max_t(int, val * 2, SOCK_MIN_RCVBUF);
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700798 break;
799
800 case SO_RCVBUFFORCE:
801 if (!capable(CAP_NET_ADMIN)) {
802 ret = -EPERM;
803 break;
804 }
805 goto set_rcvbuf;
806
807 case SO_KEEPALIVE:
Ursula Braun4b9d07a2017-01-09 16:55:12 +0100808 if (sk->sk_prot->keepalive)
809 sk->sk_prot->keepalive(sk, valbool);
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700810 sock_valbool_flag(sk, SOCK_KEEPOPEN, valbool);
811 break;
812
813 case SO_OOBINLINE:
814 sock_valbool_flag(sk, SOCK_URGINLINE, valbool);
815 break;
816
817 case SO_NO_CHECK:
Tom Herbert28448b82014-05-23 08:47:19 -0700818 sk->sk_no_check_tx = valbool;
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700819 break;
820
821 case SO_PRIORITY:
Eric W. Biederman5e1fccc2012-11-16 03:03:04 +0000822 if ((val >= 0 && val <= 6) ||
823 ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700824 sk->sk_priority = val;
825 else
826 ret = -EPERM;
827 break;
828
829 case SO_LINGER:
830 if (optlen < sizeof(ling)) {
831 ret = -EINVAL; /* 1003.1g */
832 break;
833 }
Eric Dumazet2a915252009-05-27 11:30:05 +0000834 if (copy_from_user(&ling, optval, sizeof(ling))) {
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700835 ret = -EFAULT;
836 break;
837 }
838 if (!ling.l_onoff)
839 sock_reset_flag(sk, SOCK_LINGER);
840 else {
841#if (BITS_PER_LONG == 32)
842 if ((unsigned int)ling.l_linger >= MAX_SCHEDULE_TIMEOUT/HZ)
843 sk->sk_lingertime = MAX_SCHEDULE_TIMEOUT;
844 else
845#endif
846 sk->sk_lingertime = (unsigned int)ling.l_linger * HZ;
847 sock_set_flag(sk, SOCK_LINGER);
848 }
849 break;
850
851 case SO_BSDCOMPAT:
852 sock_warn_obsolete_bsdism("setsockopt");
853 break;
854
855 case SO_PASSCRED:
856 if (valbool)
857 set_bit(SOCK_PASSCRED, &sock->flags);
858 else
859 clear_bit(SOCK_PASSCRED, &sock->flags);
860 break;
861
862 case SO_TIMESTAMP:
Eric Dumazet92f37fd2007-03-25 22:14:49 -0700863 case SO_TIMESTAMPNS:
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700864 if (valbool) {
Eric Dumazet92f37fd2007-03-25 22:14:49 -0700865 if (optname == SO_TIMESTAMP)
866 sock_reset_flag(sk, SOCK_RCVTSTAMPNS);
867 else
868 sock_set_flag(sk, SOCK_RCVTSTAMPNS);
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700869 sock_set_flag(sk, SOCK_RCVTSTAMP);
Patrick Ohly20d49472009-02-12 05:03:38 +0000870 sock_enable_timestamp(sk, SOCK_TIMESTAMP);
Eric Dumazet92f37fd2007-03-25 22:14:49 -0700871 } else {
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700872 sock_reset_flag(sk, SOCK_RCVTSTAMP);
Eric Dumazet92f37fd2007-03-25 22:14:49 -0700873 sock_reset_flag(sk, SOCK_RCVTSTAMPNS);
874 }
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700875 break;
876
Patrick Ohly20d49472009-02-12 05:03:38 +0000877 case SO_TIMESTAMPING:
878 if (val & ~SOF_TIMESTAMPING_MASK) {
Rémi Denis-Courmontf249fb72009-07-20 00:47:04 +0000879 ret = -EINVAL;
Patrick Ohly20d49472009-02-12 05:03:38 +0000880 break;
881 }
Willem de Bruijnb245be12015-01-30 13:29:32 -0500882
Willem de Bruijn09c2d252014-08-04 22:11:47 -0400883 if (val & SOF_TIMESTAMPING_OPT_ID &&
Willem de Bruijn4ed2d762014-08-04 22:11:49 -0400884 !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)) {
WANG Congac5cc972015-12-16 23:39:04 -0800885 if (sk->sk_protocol == IPPROTO_TCP &&
886 sk->sk_type == SOCK_STREAM) {
Soheil Hassas Yeganeh6db8b962016-04-02 23:08:07 -0400887 if ((1 << sk->sk_state) &
888 (TCPF_CLOSE | TCPF_LISTEN)) {
Willem de Bruijn4ed2d762014-08-04 22:11:49 -0400889 ret = -EINVAL;
890 break;
891 }
892 sk->sk_tskey = tcp_sk(sk)->snd_una;
893 } else {
894 sk->sk_tskey = 0;
895 }
896 }
Francis Yan1c885802016-11-27 23:07:18 -0800897
898 if (val & SOF_TIMESTAMPING_OPT_STATS &&
899 !(val & SOF_TIMESTAMPING_OPT_TSONLY)) {
900 ret = -EINVAL;
901 break;
902 }
903
Willem de Bruijnb9f40e22014-08-04 22:11:46 -0400904 sk->sk_tsflags = val;
Patrick Ohly20d49472009-02-12 05:03:38 +0000905 if (val & SOF_TIMESTAMPING_RX_SOFTWARE)
906 sock_enable_timestamp(sk,
907 SOCK_TIMESTAMPING_RX_SOFTWARE);
908 else
909 sock_disable_timestamp(sk,
Eric Dumazet08e29af2011-11-28 12:04:18 +0000910 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE));
Patrick Ohly20d49472009-02-12 05:03:38 +0000911 break;
912
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700913 case SO_RCVLOWAT:
914 if (val < 0)
915 val = INT_MAX;
916 sk->sk_rcvlowat = val ? : 1;
917 break;
918
919 case SO_RCVTIMEO:
920 ret = sock_set_timeout(&sk->sk_rcvtimeo, optval, optlen);
921 break;
922
923 case SO_SNDTIMEO:
924 ret = sock_set_timeout(&sk->sk_sndtimeo, optval, optlen);
925 break;
926
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700927 case SO_ATTACH_FILTER:
928 ret = -EINVAL;
929 if (optlen == sizeof(struct sock_fprog)) {
930 struct sock_fprog fprog;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700931
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700932 ret = -EFAULT;
933 if (copy_from_user(&fprog, optval, sizeof(fprog)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700934 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700935
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700936 ret = sk_attach_filter(&fprog, sk);
937 }
938 break;
939
Alexei Starovoitov89aa0752014-12-01 15:06:35 -0800940 case SO_ATTACH_BPF:
941 ret = -EINVAL;
942 if (optlen == sizeof(u32)) {
943 u32 ufd;
944
945 ret = -EFAULT;
946 if (copy_from_user(&ufd, optval, sizeof(ufd)))
947 break;
948
949 ret = sk_attach_bpf(ufd, sk);
950 }
951 break;
952
Craig Gallek538950a2016-01-04 17:41:47 -0500953 case SO_ATTACH_REUSEPORT_CBPF:
954 ret = -EINVAL;
955 if (optlen == sizeof(struct sock_fprog)) {
956 struct sock_fprog fprog;
957
958 ret = -EFAULT;
959 if (copy_from_user(&fprog, optval, sizeof(fprog)))
960 break;
961
962 ret = sk_reuseport_attach_filter(&fprog, sk);
963 }
964 break;
965
966 case SO_ATTACH_REUSEPORT_EBPF:
967 ret = -EINVAL;
968 if (optlen == sizeof(u32)) {
969 u32 ufd;
970
971 ret = -EFAULT;
972 if (copy_from_user(&ufd, optval, sizeof(ufd)))
973 break;
974
975 ret = sk_reuseport_attach_bpf(ufd, sk);
976 }
977 break;
978
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700979 case SO_DETACH_FILTER:
Pavel Emelyanov55b33322007-10-17 21:21:26 -0700980 ret = sk_detach_filter(sk);
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700981 break;
982
Vincent Bernatd59577b2013-01-16 22:55:49 +0100983 case SO_LOCK_FILTER:
984 if (sock_flag(sk, SOCK_FILTER_LOCKED) && !valbool)
985 ret = -EPERM;
986 else
987 sock_valbool_flag(sk, SOCK_FILTER_LOCKED, valbool);
988 break;
989
Stephen Hemmingere71a4782007-04-10 20:10:33 -0700990 case SO_PASSSEC:
991 if (valbool)
992 set_bit(SOCK_PASSSEC, &sock->flags);
993 else
994 clear_bit(SOCK_PASSSEC, &sock->flags);
995 break;
Laszlo Attila Toth4a19ec52008-01-30 19:08:16 -0800996 case SO_MARK:
Eric W. Biederman5e1fccc2012-11-16 03:03:04 +0000997 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
Laszlo Attila Toth4a19ec52008-01-30 19:08:16 -0800998 ret = -EPERM;
Eric Dumazet2a915252009-05-27 11:30:05 +0000999 else
Laszlo Attila Toth4a19ec52008-01-30 19:08:16 -08001000 sk->sk_mark = val;
Laszlo Attila Toth4a19ec52008-01-30 19:08:16 -08001001 break;
Catherine Zhang877ce7c2006-06-29 12:27:47 -07001002
Neil Horman3b885782009-10-12 13:26:31 -07001003 case SO_RXQ_OVFL:
Johannes Berg8083f0f2011-10-07 03:30:20 +00001004 sock_valbool_flag(sk, SOCK_RXQ_OVFL, valbool);
Neil Horman3b885782009-10-12 13:26:31 -07001005 break;
Johannes Berg6e3e9392011-11-09 10:15:42 +01001006
1007 case SO_WIFI_STATUS:
1008 sock_valbool_flag(sk, SOCK_WIFI_STATUS, valbool);
1009 break;
1010
Pavel Emelyanovef64a542012-02-21 07:31:34 +00001011 case SO_PEEK_OFF:
1012 if (sock->ops->set_peek_off)
Sasha Levin12663bf2013-12-07 17:26:27 -05001013 ret = sock->ops->set_peek_off(sk, val);
Pavel Emelyanovef64a542012-02-21 07:31:34 +00001014 else
1015 ret = -EOPNOTSUPP;
1016 break;
Ben Greear3bdc0eb2012-02-11 15:39:30 +00001017
1018 case SO_NOFCS:
1019 sock_valbool_flag(sk, SOCK_NOFCS, valbool);
1020 break;
1021
Keller, Jacob E7d4c04f2013-03-28 11:19:25 +00001022 case SO_SELECT_ERR_QUEUE:
1023 sock_valbool_flag(sk, SOCK_SELECT_ERR_QUEUE, valbool);
1024 break;
1025
Cong Wange0d10952013-08-01 11:10:25 +08001026#ifdef CONFIG_NET_RX_BUSY_POLL
Eliezer Tamir64b0dc52013-07-10 17:13:36 +03001027 case SO_BUSY_POLL:
Eliezer Tamirdafcc432013-06-14 16:33:57 +03001028 /* allow unprivileged users to decrease the value */
1029 if ((val > sk->sk_ll_usec) && !capable(CAP_NET_ADMIN))
1030 ret = -EPERM;
1031 else {
1032 if (val < 0)
1033 ret = -EINVAL;
1034 else
1035 sk->sk_ll_usec = val;
1036 }
1037 break;
1038#endif
Eric Dumazet62748f32013-09-24 08:20:52 -07001039
1040 case SO_MAX_PACING_RATE:
Eric Dumazet218af592017-05-16 04:24:36 -07001041 if (val != ~0U)
1042 cmpxchg(&sk->sk_pacing_status,
1043 SK_PACING_NONE,
1044 SK_PACING_NEEDED);
Eric Dumazet62748f32013-09-24 08:20:52 -07001045 sk->sk_max_pacing_rate = val;
1046 sk->sk_pacing_rate = min(sk->sk_pacing_rate,
1047 sk->sk_max_pacing_rate);
1048 break;
1049
Eric Dumazet70da2682015-10-08 19:33:21 -07001050 case SO_INCOMING_CPU:
1051 sk->sk_incoming_cpu = val;
1052 break;
1053
Tom Herberta87cb3e2016-02-24 10:02:52 -08001054 case SO_CNX_ADVICE:
1055 if (val == 1)
1056 dst_negative_advice(sk);
1057 break;
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001058 default:
1059 ret = -ENOPROTOOPT;
1060 break;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001061 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001062 release_sock(sk);
1063 return ret;
1064}
Eric Dumazet2a915252009-05-27 11:30:05 +00001065EXPORT_SYMBOL(sock_setsockopt);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001066
1067
stephen hemminger8f098982014-01-03 09:17:14 -08001068static void cred_to_ucred(struct pid *pid, const struct cred *cred,
1069 struct ucred *ucred)
Eric W. Biederman3f551f92010-06-13 03:28:59 +00001070{
1071 ucred->pid = pid_vnr(pid);
1072 ucred->uid = ucred->gid = -1;
1073 if (cred) {
1074 struct user_namespace *current_ns = current_user_ns();
1075
Eric W. Biedermanb2e4f542012-05-23 16:39:45 -06001076 ucred->uid = from_kuid_munged(current_ns, cred->euid);
1077 ucred->gid = from_kgid_munged(current_ns, cred->egid);
Eric W. Biederman3f551f92010-06-13 03:28:59 +00001078 }
1079}
1080
Linus Torvalds1da177e2005-04-16 15:20:36 -07001081int sock_getsockopt(struct socket *sock, int level, int optname,
1082 char __user *optval, int __user *optlen)
1083{
1084 struct sock *sk = sock->sk;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001085
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001086 union {
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001087 int val;
Chenbo Feng5daab9d2017-04-05 19:00:55 -07001088 u64 val64;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001089 struct linger ling;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001090 struct timeval tm;
1091 } v;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001092
H Hartley Sweeten4d0392b2010-01-15 01:08:58 -08001093 int lv = sizeof(int);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001094 int len;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001095
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001096 if (get_user(len, optlen))
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001097 return -EFAULT;
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001098 if (len < 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001099 return -EINVAL;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001100
Eugene Teo50fee1d2009-02-23 15:38:41 -08001101 memset(&v, 0, sizeof(v));
Clément Lecignedf0bca02009-02-12 16:59:09 -08001102
Eric Dumazet2a915252009-05-27 11:30:05 +00001103 switch (optname) {
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001104 case SO_DEBUG:
1105 v.val = sock_flag(sk, SOCK_DBG);
1106 break;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001107
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001108 case SO_DONTROUTE:
1109 v.val = sock_flag(sk, SOCK_LOCALROUTE);
1110 break;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001111
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001112 case SO_BROADCAST:
Eric Dumazet1b23a5d2012-05-16 05:57:07 +00001113 v.val = sock_flag(sk, SOCK_BROADCAST);
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001114 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001115
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001116 case SO_SNDBUF:
1117 v.val = sk->sk_sndbuf;
1118 break;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001119
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001120 case SO_RCVBUF:
1121 v.val = sk->sk_rcvbuf;
1122 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001123
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001124 case SO_REUSEADDR:
1125 v.val = sk->sk_reuse;
1126 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001127
Tom Herbert055dc212013-01-22 09:49:50 +00001128 case SO_REUSEPORT:
1129 v.val = sk->sk_reuseport;
1130 break;
1131
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001132 case SO_KEEPALIVE:
Eric Dumazet1b23a5d2012-05-16 05:57:07 +00001133 v.val = sock_flag(sk, SOCK_KEEPOPEN);
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001134 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001135
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001136 case SO_TYPE:
1137 v.val = sk->sk_type;
1138 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001139
Jan Engelhardt49c794e2009-08-04 07:28:28 +00001140 case SO_PROTOCOL:
1141 v.val = sk->sk_protocol;
1142 break;
1143
Jan Engelhardt0d6038e2009-08-04 07:28:29 +00001144 case SO_DOMAIN:
1145 v.val = sk->sk_family;
1146 break;
1147
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001148 case SO_ERROR:
1149 v.val = -sock_error(sk);
Eric Dumazet2a915252009-05-27 11:30:05 +00001150 if (v.val == 0)
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001151 v.val = xchg(&sk->sk_err_soft, 0);
1152 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001153
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001154 case SO_OOBINLINE:
Eric Dumazet1b23a5d2012-05-16 05:57:07 +00001155 v.val = sock_flag(sk, SOCK_URGINLINE);
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001156 break;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001157
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001158 case SO_NO_CHECK:
Tom Herbert28448b82014-05-23 08:47:19 -07001159 v.val = sk->sk_no_check_tx;
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001160 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001161
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001162 case SO_PRIORITY:
1163 v.val = sk->sk_priority;
1164 break;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001165
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001166 case SO_LINGER:
1167 lv = sizeof(v.ling);
Eric Dumazet1b23a5d2012-05-16 05:57:07 +00001168 v.ling.l_onoff = sock_flag(sk, SOCK_LINGER);
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001169 v.ling.l_linger = sk->sk_lingertime / HZ;
1170 break;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001171
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001172 case SO_BSDCOMPAT:
1173 sock_warn_obsolete_bsdism("getsockopt");
1174 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001175
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001176 case SO_TIMESTAMP:
Eric Dumazet92f37fd2007-03-25 22:14:49 -07001177 v.val = sock_flag(sk, SOCK_RCVTSTAMP) &&
1178 !sock_flag(sk, SOCK_RCVTSTAMPNS);
1179 break;
1180
1181 case SO_TIMESTAMPNS:
1182 v.val = sock_flag(sk, SOCK_RCVTSTAMPNS);
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001183 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001184
Patrick Ohly20d49472009-02-12 05:03:38 +00001185 case SO_TIMESTAMPING:
Willem de Bruijnb9f40e22014-08-04 22:11:46 -04001186 v.val = sk->sk_tsflags;
Patrick Ohly20d49472009-02-12 05:03:38 +00001187 break;
1188
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001189 case SO_RCVTIMEO:
Eric Dumazet2a915252009-05-27 11:30:05 +00001190 lv = sizeof(struct timeval);
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001191 if (sk->sk_rcvtimeo == MAX_SCHEDULE_TIMEOUT) {
1192 v.tm.tv_sec = 0;
1193 v.tm.tv_usec = 0;
1194 } else {
1195 v.tm.tv_sec = sk->sk_rcvtimeo / HZ;
Gao Feng8ccde4c2017-02-21 17:09:19 +08001196 v.tm.tv_usec = ((sk->sk_rcvtimeo % HZ) * USEC_PER_SEC) / HZ;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001197 }
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001198 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001199
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001200 case SO_SNDTIMEO:
Eric Dumazet2a915252009-05-27 11:30:05 +00001201 lv = sizeof(struct timeval);
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001202 if (sk->sk_sndtimeo == MAX_SCHEDULE_TIMEOUT) {
1203 v.tm.tv_sec = 0;
1204 v.tm.tv_usec = 0;
1205 } else {
1206 v.tm.tv_sec = sk->sk_sndtimeo / HZ;
Gao Feng8ccde4c2017-02-21 17:09:19 +08001207 v.tm.tv_usec = ((sk->sk_sndtimeo % HZ) * USEC_PER_SEC) / HZ;
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001208 }
1209 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001210
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001211 case SO_RCVLOWAT:
1212 v.val = sk->sk_rcvlowat;
1213 break;
Catherine Zhang877ce7c2006-06-29 12:27:47 -07001214
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001215 case SO_SNDLOWAT:
Eric Dumazet2a915252009-05-27 11:30:05 +00001216 v.val = 1;
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001217 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001218
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001219 case SO_PASSCRED:
Eric Dumazet82981932012-04-26 20:07:59 +00001220 v.val = !!test_bit(SOCK_PASSCRED, &sock->flags);
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001221 break;
1222
1223 case SO_PEERCRED:
Eric W. Biederman109f6e32010-06-13 03:30:14 +00001224 {
1225 struct ucred peercred;
1226 if (len > sizeof(peercred))
1227 len = sizeof(peercred);
1228 cred_to_ucred(sk->sk_peer_pid, sk->sk_peer_cred, &peercred);
1229 if (copy_to_user(optval, &peercred, len))
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001230 return -EFAULT;
1231 goto lenout;
Eric W. Biederman109f6e32010-06-13 03:30:14 +00001232 }
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001233
1234 case SO_PEERNAME:
1235 {
1236 char address[128];
1237
1238 if (sock->ops->getname(sock, (struct sockaddr *)address, &lv, 2))
1239 return -ENOTCONN;
1240 if (lv < len)
1241 return -EINVAL;
1242 if (copy_to_user(optval, address, len))
1243 return -EFAULT;
1244 goto lenout;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001245 }
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001246
1247 /* Dubious BSD thing... Probably nobody even uses it, but
1248 * the UNIX standard wants it for whatever reason... -DaveM
1249 */
1250 case SO_ACCEPTCONN:
1251 v.val = sk->sk_state == TCP_LISTEN;
1252 break;
1253
1254 case SO_PASSSEC:
Eric Dumazet82981932012-04-26 20:07:59 +00001255 v.val = !!test_bit(SOCK_PASSSEC, &sock->flags);
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001256 break;
1257
1258 case SO_PEERSEC:
1259 return security_socket_getpeersec_stream(sock, optval, optlen, len);
1260
Laszlo Attila Toth4a19ec52008-01-30 19:08:16 -08001261 case SO_MARK:
1262 v.val = sk->sk_mark;
1263 break;
1264
Neil Horman3b885782009-10-12 13:26:31 -07001265 case SO_RXQ_OVFL:
Eric Dumazet1b23a5d2012-05-16 05:57:07 +00001266 v.val = sock_flag(sk, SOCK_RXQ_OVFL);
Neil Horman3b885782009-10-12 13:26:31 -07001267 break;
1268
Johannes Berg6e3e9392011-11-09 10:15:42 +01001269 case SO_WIFI_STATUS:
Eric Dumazet1b23a5d2012-05-16 05:57:07 +00001270 v.val = sock_flag(sk, SOCK_WIFI_STATUS);
Johannes Berg6e3e9392011-11-09 10:15:42 +01001271 break;
1272
Pavel Emelyanovef64a542012-02-21 07:31:34 +00001273 case SO_PEEK_OFF:
1274 if (!sock->ops->set_peek_off)
1275 return -EOPNOTSUPP;
1276
1277 v.val = sk->sk_peek_off;
1278 break;
David S. Millerbc2f7992012-02-24 14:48:34 -05001279 case SO_NOFCS:
Eric Dumazet1b23a5d2012-05-16 05:57:07 +00001280 v.val = sock_flag(sk, SOCK_NOFCS);
David S. Millerbc2f7992012-02-24 14:48:34 -05001281 break;
Brian Haleyc91f6df2012-11-26 05:21:08 +00001282
Pavel Emelyanovf7b86bf2012-10-18 23:55:56 +00001283 case SO_BINDTODEVICE:
Brian Haleyc91f6df2012-11-26 05:21:08 +00001284 return sock_getbindtodevice(sk, optval, optlen, len);
1285
Pavel Emelyanova8fc9272012-11-01 02:01:48 +00001286 case SO_GET_FILTER:
1287 len = sk_get_filter(sk, (struct sock_filter __user *)optval, len);
1288 if (len < 0)
1289 return len;
1290
1291 goto lenout;
Brian Haleyc91f6df2012-11-26 05:21:08 +00001292
Vincent Bernatd59577b2013-01-16 22:55:49 +01001293 case SO_LOCK_FILTER:
1294 v.val = sock_flag(sk, SOCK_FILTER_LOCKED);
1295 break;
1296
Michal Sekletarea02f942014-01-17 17:09:45 +01001297 case SO_BPF_EXTENSIONS:
1298 v.val = bpf_tell_extensions();
1299 break;
1300
Keller, Jacob E7d4c04f2013-03-28 11:19:25 +00001301 case SO_SELECT_ERR_QUEUE:
1302 v.val = sock_flag(sk, SOCK_SELECT_ERR_QUEUE);
1303 break;
1304
Cong Wange0d10952013-08-01 11:10:25 +08001305#ifdef CONFIG_NET_RX_BUSY_POLL
Eliezer Tamir64b0dc52013-07-10 17:13:36 +03001306 case SO_BUSY_POLL:
Eliezer Tamirdafcc432013-06-14 16:33:57 +03001307 v.val = sk->sk_ll_usec;
1308 break;
1309#endif
1310
Eric Dumazet62748f32013-09-24 08:20:52 -07001311 case SO_MAX_PACING_RATE:
1312 v.val = sk->sk_max_pacing_rate;
1313 break;
1314
Eric Dumazet2c8c56e2014-11-11 05:54:28 -08001315 case SO_INCOMING_CPU:
1316 v.val = sk->sk_incoming_cpu;
1317 break;
1318
Josh Hunta2d133b2017-03-20 15:22:03 -04001319 case SO_MEMINFO:
1320 {
1321 u32 meminfo[SK_MEMINFO_VARS];
1322
1323 if (get_user(len, optlen))
1324 return -EFAULT;
1325
1326 sk_get_meminfo(sk, meminfo);
1327
1328 len = min_t(unsigned int, len, sizeof(meminfo));
1329 if (copy_to_user(optval, &meminfo, len))
1330 return -EFAULT;
1331
1332 goto lenout;
1333 }
Sridhar Samudrala6d433902017-03-24 10:08:36 -07001334
1335#ifdef CONFIG_NET_RX_BUSY_POLL
1336 case SO_INCOMING_NAPI_ID:
1337 v.val = READ_ONCE(sk->sk_napi_id);
1338
1339 /* aggregate non-NAPI IDs down to 0 */
1340 if (v.val < MIN_NAPI_ID)
1341 v.val = 0;
1342
1343 break;
1344#endif
1345
Chenbo Feng5daab9d2017-04-05 19:00:55 -07001346 case SO_COOKIE:
1347 lv = sizeof(u64);
1348 if (len < lv)
1349 return -EINVAL;
1350 v.val64 = sock_gen_cookie(sk);
1351 break;
1352
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001353 default:
YOSHIFUJI Hideaki/吉藤英明443b5992015-03-23 18:04:13 +09001354 /* We implement the SO_SNDLOWAT etc to not be settable
1355 * (1003.1g 7).
1356 */
Stephen Hemmingere71a4782007-04-10 20:10:33 -07001357 return -ENOPROTOOPT;
1358 }
1359
Linus Torvalds1da177e2005-04-16 15:20:36 -07001360 if (len > lv)
1361 len = lv;
1362 if (copy_to_user(optval, &v, len))
1363 return -EFAULT;
1364lenout:
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001365 if (put_user(len, optlen))
1366 return -EFAULT;
1367 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001368}
1369
Ingo Molnara5b5bb92006-07-03 00:25:35 -07001370/*
1371 * Initialize an sk_lock.
1372 *
1373 * (We also register the sk_lock with the lock validator.)
1374 */
Dave Jonesb6f99a22007-03-22 12:27:49 -07001375static inline void sock_lock_init(struct sock *sk)
Ingo Molnara5b5bb92006-07-03 00:25:35 -07001376{
David Howellscdfbabf2017-03-09 08:09:05 +00001377 if (sk->sk_kern_sock)
1378 sock_lock_init_class_and_name(
1379 sk,
1380 af_family_kern_slock_key_strings[sk->sk_family],
1381 af_family_kern_slock_keys + sk->sk_family,
1382 af_family_kern_key_strings[sk->sk_family],
1383 af_family_kern_keys + sk->sk_family);
1384 else
1385 sock_lock_init_class_and_name(
1386 sk,
Peter Zijlstraed075362006-12-06 20:35:24 -08001387 af_family_slock_key_strings[sk->sk_family],
1388 af_family_slock_keys + sk->sk_family,
1389 af_family_key_strings[sk->sk_family],
1390 af_family_keys + sk->sk_family);
Ingo Molnara5b5bb92006-07-03 00:25:35 -07001391}
1392
Eric Dumazet4dc6dc72009-07-15 23:13:10 +00001393/*
1394 * Copy all fields from osk to nsk but nsk->sk_refcnt must not change yet,
1395 * even temporarly, because of RCU lookups. sk_node should also be left as is.
Eric Dumazet68835ab2010-11-30 19:04:07 +00001396 * We must not copy fields between sk_dontcopy_begin and sk_dontcopy_end
Eric Dumazet4dc6dc72009-07-15 23:13:10 +00001397 */
Pavel Emelyanovf1a6c4d2007-11-01 00:29:45 -07001398static void sock_copy(struct sock *nsk, const struct sock *osk)
1399{
1400#ifdef CONFIG_SECURITY_NETWORK
1401 void *sptr = nsk->sk_security;
1402#endif
Eric Dumazet68835ab2010-11-30 19:04:07 +00001403 memcpy(nsk, osk, offsetof(struct sock, sk_dontcopy_begin));
1404
1405 memcpy(&nsk->sk_dontcopy_end, &osk->sk_dontcopy_end,
1406 osk->sk_prot->obj_size - offsetof(struct sock, sk_dontcopy_end));
1407
Pavel Emelyanovf1a6c4d2007-11-01 00:29:45 -07001408#ifdef CONFIG_SECURITY_NETWORK
1409 nsk->sk_security = sptr;
1410 security_sk_clone(osk, nsk);
1411#endif
1412}
1413
Pavel Emelyanov2e4afe72007-11-01 00:36:26 -07001414static struct sock *sk_prot_alloc(struct proto *prot, gfp_t priority,
1415 int family)
Pavel Emelyanovc308c1b22007-11-01 00:33:50 -07001416{
1417 struct sock *sk;
1418 struct kmem_cache *slab;
1419
1420 slab = prot->slab;
Eric Dumazete912b112009-07-08 19:36:05 +00001421 if (slab != NULL) {
1422 sk = kmem_cache_alloc(slab, priority & ~__GFP_ZERO);
1423 if (!sk)
1424 return sk;
Eric Dumazetba2489b2016-08-23 11:39:29 -07001425 if (priority & __GFP_ZERO)
1426 sk_prot_clear_nulls(sk, prot->obj_size);
Octavian Purdilafcbdf092010-12-16 14:26:56 -08001427 } else
Pavel Emelyanovc308c1b22007-11-01 00:33:50 -07001428 sk = kmalloc(prot->obj_size, priority);
1429
Pavel Emelyanov2e4afe72007-11-01 00:36:26 -07001430 if (sk != NULL) {
Vegard Nossuma98b65a2009-02-26 14:46:57 +01001431 kmemcheck_annotate_bitfield(sk, flags);
1432
Pavel Emelyanov2e4afe72007-11-01 00:36:26 -07001433 if (security_sk_alloc(sk, family, priority))
1434 goto out_free;
1435
1436 if (!try_module_get(prot->owner))
1437 goto out_free_sec;
Krishna Kumare022f0b2009-10-19 23:46:20 +00001438 sk_tx_queue_clear(sk);
Pavel Emelyanov2e4afe72007-11-01 00:36:26 -07001439 }
1440
Pavel Emelyanovc308c1b22007-11-01 00:33:50 -07001441 return sk;
Pavel Emelyanov2e4afe72007-11-01 00:36:26 -07001442
1443out_free_sec:
1444 security_sk_free(sk);
1445out_free:
1446 if (slab != NULL)
1447 kmem_cache_free(slab, sk);
1448 else
1449 kfree(sk);
1450 return NULL;
Pavel Emelyanovc308c1b22007-11-01 00:33:50 -07001451}
1452
1453static void sk_prot_free(struct proto *prot, struct sock *sk)
1454{
1455 struct kmem_cache *slab;
Pavel Emelyanov2e4afe72007-11-01 00:36:26 -07001456 struct module *owner;
Pavel Emelyanovc308c1b22007-11-01 00:33:50 -07001457
Pavel Emelyanov2e4afe72007-11-01 00:36:26 -07001458 owner = prot->owner;
Pavel Emelyanovc308c1b22007-11-01 00:33:50 -07001459 slab = prot->slab;
Pavel Emelyanov2e4afe72007-11-01 00:36:26 -07001460
Tejun Heobd1060a2015-12-07 17:38:53 -05001461 cgroup_sk_free(&sk->sk_cgrp_data);
Johannes Weiner2d758072016-10-07 17:00:58 -07001462 mem_cgroup_sk_free(sk);
Pavel Emelyanov2e4afe72007-11-01 00:36:26 -07001463 security_sk_free(sk);
Pavel Emelyanovc308c1b22007-11-01 00:33:50 -07001464 if (slab != NULL)
1465 kmem_cache_free(slab, sk);
1466 else
1467 kfree(sk);
Pavel Emelyanov2e4afe72007-11-01 00:36:26 -07001468 module_put(owner);
Pavel Emelyanovc308c1b22007-11-01 00:33:50 -07001469}
1470
Linus Torvalds1da177e2005-04-16 15:20:36 -07001471/**
1472 * sk_alloc - All socket objects are allocated here
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07001473 * @net: the applicable net namespace
Pavel Pisa4dc3b162005-05-01 08:59:25 -07001474 * @family: protocol family
1475 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc)
1476 * @prot: struct proto associated with this new sock instance
Eric W. Biederman11aa9c22015-05-08 21:09:13 -05001477 * @kern: is this to be a kernel socket?
Linus Torvalds1da177e2005-04-16 15:20:36 -07001478 */
Eric W. Biederman1b8d7ae2007-10-08 23:24:22 -07001479struct sock *sk_alloc(struct net *net, int family, gfp_t priority,
Eric W. Biederman11aa9c22015-05-08 21:09:13 -05001480 struct proto *prot, int kern)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001481{
Pavel Emelyanovc308c1b22007-11-01 00:33:50 -07001482 struct sock *sk;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001483
Pavel Emelyanov154adbc2007-11-01 00:38:43 -07001484 sk = sk_prot_alloc(prot, priority | __GFP_ZERO, family);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001485 if (sk) {
Pavel Emelyanov154adbc2007-11-01 00:38:43 -07001486 sk->sk_family = family;
1487 /*
1488 * See comment in struct sock definition to understand
1489 * why we need sk_prot_creator -acme
1490 */
1491 sk->sk_prot = sk->sk_prot_creator = prot;
David Howellscdfbabf2017-03-09 08:09:05 +00001492 sk->sk_kern_sock = kern;
Pavel Emelyanov154adbc2007-11-01 00:38:43 -07001493 sock_lock_init(sk);
Eric W. Biederman26abe142015-05-08 21:10:31 -05001494 sk->sk_net_refcnt = kern ? 0 : 1;
1495 if (likely(sk->sk_net_refcnt))
1496 get_net(net);
1497 sock_net_set(sk, net);
Jarek Poplawskid66ee052009-08-30 23:15:36 +00001498 atomic_set(&sk->sk_wmem_alloc, 1);
Herbert Xuf8451722010-05-24 00:12:34 -07001499
Johannes Weiner2d758072016-10-07 17:00:58 -07001500 mem_cgroup_sk_alloc(sk);
Johannes Weinerd979a392016-09-19 14:44:38 -07001501 cgroup_sk_alloc(&sk->sk_cgrp_data);
Tejun Heo2a56a1f2015-12-07 17:38:52 -05001502 sock_update_classid(&sk->sk_cgrp_data);
1503 sock_update_netprioidx(&sk->sk_cgrp_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001504 }
Frank Filza79af592005-09-27 15:23:38 -07001505
Pavel Emelyanov2e4afe72007-11-01 00:36:26 -07001506 return sk;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001507}
Eric Dumazet2a915252009-05-27 11:30:05 +00001508EXPORT_SYMBOL(sk_alloc);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001509
Eric Dumazeta4298e42016-04-01 08:52:12 -07001510/* Sockets having SOCK_RCU_FREE will call this function after one RCU
1511 * grace period. This is the case for UDP sockets and TCP listeners.
1512 */
1513static void __sk_destruct(struct rcu_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001514{
Eric Dumazeta4298e42016-04-01 08:52:12 -07001515 struct sock *sk = container_of(head, struct sock, sk_rcu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001516 struct sk_filter *filter;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001517
1518 if (sk->sk_destruct)
1519 sk->sk_destruct(sk);
1520
Paul E. McKenneya898def2010-02-22 17:04:49 -08001521 filter = rcu_dereference_check(sk->sk_filter,
1522 atomic_read(&sk->sk_wmem_alloc) == 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001523 if (filter) {
Pavel Emelyanov309dd5f2007-10-17 21:21:51 -07001524 sk_filter_uncharge(sk, filter);
Stephen Hemmingera9b3cd72011-08-01 16:19:00 +00001525 RCU_INIT_POINTER(sk->sk_filter, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001526 }
Craig Gallek538950a2016-01-04 17:41:47 -05001527 if (rcu_access_pointer(sk->sk_reuseport_cb))
1528 reuseport_detach_sock(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001529
Eric Dumazet08e29af2011-11-28 12:04:18 +00001530 sock_disable_timestamp(sk, SK_FLAGS_TIMESTAMP);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001531
1532 if (atomic_read(&sk->sk_omem_alloc))
Joe Perchese005d192012-05-16 19:58:40 +00001533 pr_debug("%s: optmem leakage (%d bytes) detected\n",
1534 __func__, atomic_read(&sk->sk_omem_alloc));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001535
Eric Dumazet22a0e182017-03-15 13:21:28 -07001536 if (sk->sk_frag.page) {
1537 put_page(sk->sk_frag.page);
1538 sk->sk_frag.page = NULL;
1539 }
1540
Eric W. Biederman109f6e32010-06-13 03:30:14 +00001541 if (sk->sk_peer_cred)
1542 put_cred(sk->sk_peer_cred);
1543 put_pid(sk->sk_peer_pid);
Eric W. Biederman26abe142015-05-08 21:10:31 -05001544 if (likely(sk->sk_net_refcnt))
1545 put_net(sock_net(sk));
Pavel Emelyanovc308c1b22007-11-01 00:33:50 -07001546 sk_prot_free(sk->sk_prot_creator, sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001547}
Eric Dumazet2b85a342009-06-11 02:55:43 -07001548
Eric Dumazeta4298e42016-04-01 08:52:12 -07001549void sk_destruct(struct sock *sk)
1550{
1551 if (sock_flag(sk, SOCK_RCU_FREE))
1552 call_rcu(&sk->sk_rcu, __sk_destruct);
1553 else
1554 __sk_destruct(&sk->sk_rcu);
1555}
1556
Craig Gallekeb4cb002015-06-15 11:26:18 -04001557static void __sk_free(struct sock *sk)
1558{
Craig Gallekb9226222015-06-30 12:49:32 -04001559 if (unlikely(sock_diag_has_destroy_listeners(sk) && sk->sk_net_refcnt))
Craig Gallekeb4cb002015-06-15 11:26:18 -04001560 sock_diag_broadcast_destroy(sk);
1561 else
1562 sk_destruct(sk);
1563}
1564
Eric Dumazet2b85a342009-06-11 02:55:43 -07001565void sk_free(struct sock *sk)
1566{
1567 /*
Lucas De Marchi25985ed2011-03-30 22:57:33 -03001568 * We subtract one from sk_wmem_alloc and can know if
Eric Dumazet2b85a342009-06-11 02:55:43 -07001569 * some packets are still in some tx queue.
1570 * If not null, sock_wfree() will call __sk_free(sk) later
1571 */
1572 if (atomic_dec_and_test(&sk->sk_wmem_alloc))
1573 __sk_free(sk);
1574}
Eric Dumazet2a915252009-05-27 11:30:05 +00001575EXPORT_SYMBOL(sk_free);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001576
Paolo Abeni581319c2017-03-09 13:54:08 +01001577static void sk_init_common(struct sock *sk)
1578{
1579 skb_queue_head_init(&sk->sk_receive_queue);
1580 skb_queue_head_init(&sk->sk_write_queue);
1581 skb_queue_head_init(&sk->sk_error_queue);
1582
1583 rwlock_init(&sk->sk_callback_lock);
1584 lockdep_set_class_and_name(&sk->sk_receive_queue.lock,
1585 af_rlock_keys + sk->sk_family,
1586 af_family_rlock_key_strings[sk->sk_family]);
1587 lockdep_set_class_and_name(&sk->sk_write_queue.lock,
1588 af_wlock_keys + sk->sk_family,
1589 af_family_wlock_key_strings[sk->sk_family]);
1590 lockdep_set_class_and_name(&sk->sk_error_queue.lock,
1591 af_elock_keys + sk->sk_family,
1592 af_family_elock_key_strings[sk->sk_family]);
1593 lockdep_set_class_and_name(&sk->sk_callback_lock,
1594 af_callback_keys + sk->sk_family,
1595 af_family_clock_key_strings[sk->sk_family]);
1596}
1597
Eric Dumazete56c57d2011-11-08 17:07:07 -05001598/**
1599 * sk_clone_lock - clone a socket, and lock its clone
1600 * @sk: the socket to clone
1601 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc)
1602 *
1603 * Caller must unlock socket even in error path (bh_unlock_sock(newsk))
1604 */
1605struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority)
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001606{
Pavel Emelyanov8fd1d172007-11-01 00:37:32 -07001607 struct sock *newsk;
Alexei Starovoitov278571b2014-07-30 20:34:12 -07001608 bool is_charged = true;
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001609
Pavel Emelyanov8fd1d172007-11-01 00:37:32 -07001610 newsk = sk_prot_alloc(sk->sk_prot, priority, sk->sk_family);
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001611 if (newsk != NULL) {
1612 struct sk_filter *filter;
1613
Venkat Yekkirala892c1412006-08-04 23:08:56 -07001614 sock_copy(newsk, sk);
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001615
1616 /* SANITY */
Sowmini Varadhan8a681732015-07-30 15:50:36 +02001617 if (likely(newsk->sk_net_refcnt))
1618 get_net(sock_net(newsk));
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001619 sk_node_init(&newsk->sk_node);
1620 sock_lock_init(newsk);
1621 bh_lock_sock(newsk);
Eric Dumazetfa438cc2007-03-04 16:05:44 -08001622 newsk->sk_backlog.head = newsk->sk_backlog.tail = NULL;
Zhu Yi8eae9392010-03-04 18:01:40 +00001623 newsk->sk_backlog.len = 0;
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001624
1625 atomic_set(&newsk->sk_rmem_alloc, 0);
Eric Dumazet2b85a342009-06-11 02:55:43 -07001626 /*
1627 * sk_wmem_alloc set to one (see sk_free() and sock_wfree())
1628 */
1629 atomic_set(&newsk->sk_wmem_alloc, 1);
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001630 atomic_set(&newsk->sk_omem_alloc, 0);
Paolo Abeni581319c2017-03-09 13:54:08 +01001631 sk_init_common(newsk);
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001632
1633 newsk->sk_dst_cache = NULL;
Julian Anastasov9b8805a2017-02-06 23:14:11 +02001634 newsk->sk_dst_pending_confirm = 0;
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001635 newsk->sk_wmem_queued = 0;
1636 newsk->sk_forward_alloc = 0;
Eric Dumazet9caad862016-04-01 08:52:20 -07001637 atomic_set(&newsk->sk_drops, 0);
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001638 newsk->sk_send_head = NULL;
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001639 newsk->sk_userlocks = sk->sk_userlocks & ~SOCK_BINDPORT_LOCK;
1640
1641 sock_reset_flag(newsk, SOCK_DONE);
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001642
Eric Dumazet0d7da9d2010-10-25 03:47:05 +00001643 filter = rcu_dereference_protected(newsk->sk_filter, 1);
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001644 if (filter != NULL)
Alexei Starovoitov278571b2014-07-30 20:34:12 -07001645 /* though it's an empty new sock, the charging may fail
1646 * if sysctl_optmem_max was changed between creation of
1647 * original socket and cloning
1648 */
1649 is_charged = sk_filter_charge(newsk, filter);
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001650
Eric Dumazetd188ba82015-12-08 07:22:02 -08001651 if (unlikely(!is_charged || xfrm_sk_clone_policy(newsk, sk))) {
Daniel Borkmanna97e50c2017-03-22 13:08:08 +01001652 /* We need to make sure that we don't uncharge the new
1653 * socket if we couldn't charge it in the first place
1654 * as otherwise we uncharge the parent's filter.
1655 */
1656 if (!is_charged)
1657 RCU_INIT_POINTER(newsk->sk_filter, NULL);
Arnaldo Carvalho de Melo94352d42017-03-01 16:35:08 -03001658 sk_free_unlock_clone(newsk);
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001659 newsk = NULL;
1660 goto out;
1661 }
Craig Gallekfa463492016-02-10 11:50:39 -05001662 RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL);
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001663
1664 newsk->sk_err = 0;
Eric Dumazete551c322016-10-28 13:40:24 -07001665 newsk->sk_err_soft = 0;
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001666 newsk->sk_priority = 0;
Eric Dumazet2c8c56e2014-11-11 05:54:28 -08001667 newsk->sk_incoming_cpu = raw_smp_processor_id();
Eric Dumazet33cf7c92015-03-11 18:53:14 -07001668 atomic64_set(&newsk->sk_cookie, 0);
Johannes Weinerd979a392016-09-19 14:44:38 -07001669
Johannes Weiner2d758072016-10-07 17:00:58 -07001670 mem_cgroup_sk_alloc(newsk);
Johannes Weinerd979a392016-09-19 14:44:38 -07001671 cgroup_sk_alloc(&newsk->sk_cgrp_data);
1672
Eric Dumazet4dc6dc72009-07-15 23:13:10 +00001673 /*
1674 * Before updating sk_refcnt, we must commit prior changes to memory
1675 * (Documentation/RCU/rculist_nulls.txt for details)
1676 */
1677 smp_wmb();
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001678 atomic_set(&newsk->sk_refcnt, 2);
1679
1680 /*
1681 * Increment the counter in the same struct proto as the master
1682 * sock (sk_refcnt_debug_inc uses newsk->sk_prot->socks, that
1683 * is the same as sk->sk_prot->socks, as this field was copied
1684 * with memcpy).
1685 *
1686 * This _changes_ the previous behaviour, where
1687 * tcp_create_openreq_child always was incrementing the
1688 * equivalent to tcp_prot->socks (inet_sock_nr), so this have
1689 * to be taken into account in all callers. -acme
1690 */
1691 sk_refcnt_debug_inc(newsk);
David S. Miller972692e2008-06-17 22:41:38 -07001692 sk_set_socket(newsk, NULL);
Eric Dumazet43815482010-04-29 11:01:49 +00001693 newsk->sk_wq = NULL;
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001694
1695 if (newsk->sk_prot->sockets_allocated)
Glauber Costa180d8cd2011-12-11 21:47:02 +00001696 sk_sockets_allocated_inc(newsk);
Octavian Purdila704da5602010-01-08 00:00:09 -08001697
Hannes Frederic Sowa080a2702015-10-26 13:51:37 +01001698 if (sock_needs_netstamp(sk) &&
1699 newsk->sk_flags & SK_FLAGS_TIMESTAMP)
Octavian Purdila704da5602010-01-08 00:00:09 -08001700 net_enable_timestamp();
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001701 }
1702out:
1703 return newsk;
1704}
Eric Dumazete56c57d2011-11-08 17:07:07 -05001705EXPORT_SYMBOL_GPL(sk_clone_lock);
Arnaldo Carvalho de Melo87d11ce2005-08-09 20:10:12 -07001706
Arnaldo Carvalho de Melo94352d42017-03-01 16:35:08 -03001707void sk_free_unlock_clone(struct sock *sk)
1708{
1709 /* It is still raw copy of parent, so invalidate
1710 * destructor and make plain sk_free() */
1711 sk->sk_destruct = NULL;
1712 bh_unlock_sock(sk);
1713 sk_free(sk);
1714}
1715EXPORT_SYMBOL_GPL(sk_free_unlock_clone);
1716
Andi Kleen99580892007-04-20 17:12:43 -07001717void sk_setup_caps(struct sock *sk, struct dst_entry *dst)
1718{
Eric Dumazetd6a4e262015-05-26 08:55:28 -07001719 u32 max_segs = 1;
1720
Eric Dumazet6bd4f352015-12-02 21:53:57 -08001721 sk_dst_set(sk, dst);
Andi Kleen99580892007-04-20 17:12:43 -07001722 sk->sk_route_caps = dst->dev->features;
1723 if (sk->sk_route_caps & NETIF_F_GSO)
Herbert Xu4fcd6b92007-05-31 22:15:50 -07001724 sk->sk_route_caps |= NETIF_F_GSO_SOFTWARE;
Eric Dumazeta4654192010-05-16 00:36:33 -07001725 sk->sk_route_caps &= ~sk->sk_route_nocaps;
Andi Kleen99580892007-04-20 17:12:43 -07001726 if (sk_can_gso(sk)) {
Peter P Waskiewicz Jr82cc1a72008-03-21 03:43:19 -07001727 if (dst->header_len) {
Andi Kleen99580892007-04-20 17:12:43 -07001728 sk->sk_route_caps &= ~NETIF_F_GSO_MASK;
Peter P Waskiewicz Jr82cc1a72008-03-21 03:43:19 -07001729 } else {
Andi Kleen99580892007-04-20 17:12:43 -07001730 sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM;
Peter P Waskiewicz Jr82cc1a72008-03-21 03:43:19 -07001731 sk->sk_gso_max_size = dst->dev->gso_max_size;
Eric Dumazetd6a4e262015-05-26 08:55:28 -07001732 max_segs = max_t(u32, dst->dev->gso_max_segs, 1);
Peter P Waskiewicz Jr82cc1a72008-03-21 03:43:19 -07001733 }
Andi Kleen99580892007-04-20 17:12:43 -07001734 }
Eric Dumazetd6a4e262015-05-26 08:55:28 -07001735 sk->sk_gso_max_segs = max_segs;
Andi Kleen99580892007-04-20 17:12:43 -07001736}
1737EXPORT_SYMBOL_GPL(sk_setup_caps);
1738
Linus Torvalds1da177e2005-04-16 15:20:36 -07001739/*
1740 * Simple resource managers for sockets.
1741 */
1742
1743
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001744/*
1745 * Write buffer destructor automatically called from kfree_skb.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001746 */
1747void sock_wfree(struct sk_buff *skb)
1748{
1749 struct sock *sk = skb->sk;
Eric Dumazetd99927f2009-09-24 10:49:24 +00001750 unsigned int len = skb->truesize;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001751
Eric Dumazetd99927f2009-09-24 10:49:24 +00001752 if (!sock_flag(sk, SOCK_USE_WRITE_QUEUE)) {
1753 /*
1754 * Keep a reference on sk_wmem_alloc, this will be released
1755 * after sk_write_space() call
1756 */
1757 atomic_sub(len - 1, &sk->sk_wmem_alloc);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001758 sk->sk_write_space(sk);
Eric Dumazetd99927f2009-09-24 10:49:24 +00001759 len = 1;
1760 }
Eric Dumazet2b85a342009-06-11 02:55:43 -07001761 /*
Eric Dumazetd99927f2009-09-24 10:49:24 +00001762 * if sk_wmem_alloc reaches 0, we must finish what sk_free()
1763 * could not do because of in-flight packets
Eric Dumazet2b85a342009-06-11 02:55:43 -07001764 */
Eric Dumazetd99927f2009-09-24 10:49:24 +00001765 if (atomic_sub_and_test(len, &sk->sk_wmem_alloc))
Eric Dumazet2b85a342009-06-11 02:55:43 -07001766 __sk_free(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001767}
Eric Dumazet2a915252009-05-27 11:30:05 +00001768EXPORT_SYMBOL(sock_wfree);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001769
Eric Dumazet1d2077a2016-05-02 10:56:27 -07001770/* This variant of sock_wfree() is used by TCP,
1771 * since it sets SOCK_USE_WRITE_QUEUE.
1772 */
1773void __sock_wfree(struct sk_buff *skb)
1774{
1775 struct sock *sk = skb->sk;
1776
1777 if (atomic_sub_and_test(skb->truesize, &sk->sk_wmem_alloc))
1778 __sk_free(sk);
1779}
1780
Eric Dumazet9e17f8a2015-11-01 15:36:55 -08001781void skb_set_owner_w(struct sk_buff *skb, struct sock *sk)
1782{
1783 skb_orphan(skb);
1784 skb->sk = sk;
1785#ifdef CONFIG_INET
1786 if (unlikely(!sk_fullsock(sk))) {
1787 skb->destructor = sock_edemux;
1788 sock_hold(sk);
1789 return;
1790 }
1791#endif
1792 skb->destructor = sock_wfree;
1793 skb_set_hash_from_sk(skb, sk);
1794 /*
1795 * We used to take a refcount on sk, but following operation
1796 * is enough to guarantee sk_free() wont free this sock until
1797 * all in-flight packets are completed
1798 */
1799 atomic_add(skb->truesize, &sk->sk_wmem_alloc);
1800}
1801EXPORT_SYMBOL(skb_set_owner_w);
1802
Eric Dumazet1d2077a2016-05-02 10:56:27 -07001803/* This helper is used by netem, as it can hold packets in its
1804 * delay queue. We want to allow the owner socket to send more
1805 * packets, as if they were already TX completed by a typical driver.
1806 * But we also want to keep skb->sk set because some packet schedulers
Eric Dumazetf6ba8d32017-05-11 15:24:41 -07001807 * rely on it (sch_fq for example).
Eric Dumazet1d2077a2016-05-02 10:56:27 -07001808 */
Eric Dumazetf2f872f2013-07-30 17:55:08 -07001809void skb_orphan_partial(struct sk_buff *skb)
1810{
Eric Dumazetf6ba8d32017-05-11 15:24:41 -07001811 if (skb_is_tcp_pure_ack(skb))
Eric Dumazet1d2077a2016-05-02 10:56:27 -07001812 return;
1813
Eric Dumazetf2f872f2013-07-30 17:55:08 -07001814 if (skb->destructor == sock_wfree
1815#ifdef CONFIG_INET
1816 || skb->destructor == tcp_wfree
1817#endif
1818 ) {
Eric Dumazetf6ba8d32017-05-11 15:24:41 -07001819 struct sock *sk = skb->sk;
1820
1821 if (atomic_inc_not_zero(&sk->sk_refcnt)) {
1822 atomic_sub(skb->truesize, &sk->sk_wmem_alloc);
1823 skb->destructor = sock_efree;
1824 }
Eric Dumazetf2f872f2013-07-30 17:55:08 -07001825 } else {
1826 skb_orphan(skb);
1827 }
1828}
1829EXPORT_SYMBOL(skb_orphan_partial);
1830
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001831/*
1832 * Read buffer destructor automatically called from kfree_skb.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001833 */
1834void sock_rfree(struct sk_buff *skb)
1835{
1836 struct sock *sk = skb->sk;
Eric Dumazetd361fd52010-07-10 22:45:17 +00001837 unsigned int len = skb->truesize;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001838
Eric Dumazetd361fd52010-07-10 22:45:17 +00001839 atomic_sub(len, &sk->sk_rmem_alloc);
1840 sk_mem_uncharge(sk, len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001841}
Eric Dumazet2a915252009-05-27 11:30:05 +00001842EXPORT_SYMBOL(sock_rfree);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001843
Oliver Hartkopp7768eed2015-03-10 19:03:46 +01001844/*
1845 * Buffer destructor for skbs that are not used directly in read or write
1846 * path, e.g. for error handler skbs. Automatically called from kfree_skb.
1847 */
Alexander Duyck62bccb82014-09-04 13:31:35 -04001848void sock_efree(struct sk_buff *skb)
1849{
1850 sock_put(skb->sk);
1851}
1852EXPORT_SYMBOL(sock_efree);
1853
Eric W. Biederman976d02012012-05-23 17:16:53 -06001854kuid_t sock_i_uid(struct sock *sk)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001855{
Eric W. Biederman976d02012012-05-23 17:16:53 -06001856 kuid_t uid;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001857
Eric Dumazetf064af12010-09-22 12:43:39 +00001858 read_lock_bh(&sk->sk_callback_lock);
Eric W. Biederman976d02012012-05-23 17:16:53 -06001859 uid = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_uid : GLOBAL_ROOT_UID;
Eric Dumazetf064af12010-09-22 12:43:39 +00001860 read_unlock_bh(&sk->sk_callback_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001861 return uid;
1862}
Eric Dumazet2a915252009-05-27 11:30:05 +00001863EXPORT_SYMBOL(sock_i_uid);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001864
1865unsigned long sock_i_ino(struct sock *sk)
1866{
1867 unsigned long ino;
1868
Eric Dumazetf064af12010-09-22 12:43:39 +00001869 read_lock_bh(&sk->sk_callback_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001870 ino = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_ino : 0;
Eric Dumazetf064af12010-09-22 12:43:39 +00001871 read_unlock_bh(&sk->sk_callback_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001872 return ino;
1873}
Eric Dumazet2a915252009-05-27 11:30:05 +00001874EXPORT_SYMBOL(sock_i_ino);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001875
1876/*
1877 * Allocate a skb from the socket's send buffer.
1878 */
Victor Fusco86a76ca2005-07-08 14:57:47 -07001879struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force,
Al Virodd0fc662005-10-07 07:46:04 +01001880 gfp_t priority)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001881{
1882 if (force || atomic_read(&sk->sk_wmem_alloc) < sk->sk_sndbuf) {
Eric Dumazet2a915252009-05-27 11:30:05 +00001883 struct sk_buff *skb = alloc_skb(size, priority);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001884 if (skb) {
1885 skb_set_owner_w(skb, sk);
1886 return skb;
1887 }
1888 }
1889 return NULL;
1890}
Eric Dumazet2a915252009-05-27 11:30:05 +00001891EXPORT_SYMBOL(sock_wmalloc);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001892
1893/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001894 * Allocate a memory block from the socket's option memory buffer.
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001895 */
Al Virodd0fc662005-10-07 07:46:04 +01001896void *sock_kmalloc(struct sock *sk, int size, gfp_t priority)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001897{
Eric Dumazet95c96172012-04-15 05:58:06 +00001898 if ((unsigned int)size <= sysctl_optmem_max &&
Linus Torvalds1da177e2005-04-16 15:20:36 -07001899 atomic_read(&sk->sk_omem_alloc) + size < sysctl_optmem_max) {
1900 void *mem;
1901 /* First do the add, to avoid the race if kmalloc
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001902 * might sleep.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001903 */
1904 atomic_add(size, &sk->sk_omem_alloc);
1905 mem = kmalloc(size, priority);
1906 if (mem)
1907 return mem;
1908 atomic_sub(size, &sk->sk_omem_alloc);
1909 }
1910 return NULL;
1911}
Eric Dumazet2a915252009-05-27 11:30:05 +00001912EXPORT_SYMBOL(sock_kmalloc);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001913
Daniel Borkmann79e88652014-11-19 17:13:11 +01001914/* Free an option memory block. Note, we actually want the inline
1915 * here as this allows gcc to detect the nullify and fold away the
1916 * condition entirely.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001917 */
Daniel Borkmann79e88652014-11-19 17:13:11 +01001918static inline void __sock_kfree_s(struct sock *sk, void *mem, int size,
1919 const bool nullify)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001920{
David S. Millere53da5f2014-10-14 17:02:37 -04001921 if (WARN_ON_ONCE(!mem))
1922 return;
Daniel Borkmann79e88652014-11-19 17:13:11 +01001923 if (nullify)
1924 kzfree(mem);
1925 else
1926 kfree(mem);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001927 atomic_sub(size, &sk->sk_omem_alloc);
1928}
Daniel Borkmann79e88652014-11-19 17:13:11 +01001929
1930void sock_kfree_s(struct sock *sk, void *mem, int size)
1931{
1932 __sock_kfree_s(sk, mem, size, false);
1933}
Eric Dumazet2a915252009-05-27 11:30:05 +00001934EXPORT_SYMBOL(sock_kfree_s);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001935
Daniel Borkmann79e88652014-11-19 17:13:11 +01001936void sock_kzfree_s(struct sock *sk, void *mem, int size)
1937{
1938 __sock_kfree_s(sk, mem, size, true);
1939}
1940EXPORT_SYMBOL(sock_kzfree_s);
1941
Linus Torvalds1da177e2005-04-16 15:20:36 -07001942/* It is almost wait_for_tcp_memory minus release_sock/lock_sock.
1943 I think, these locks should be removed for datagram sockets.
1944 */
Eric Dumazet2a915252009-05-27 11:30:05 +00001945static long sock_wait_for_wmem(struct sock *sk, long timeo)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001946{
1947 DEFINE_WAIT(wait);
1948
Eric Dumazet9cd3e072015-11-29 20:03:10 -08001949 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001950 for (;;) {
1951 if (!timeo)
1952 break;
1953 if (signal_pending(current))
1954 break;
1955 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
Eric Dumazetaa395142010-04-20 13:03:51 +00001956 prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001957 if (atomic_read(&sk->sk_wmem_alloc) < sk->sk_sndbuf)
1958 break;
1959 if (sk->sk_shutdown & SEND_SHUTDOWN)
1960 break;
1961 if (sk->sk_err)
1962 break;
1963 timeo = schedule_timeout(timeo);
1964 }
Eric Dumazetaa395142010-04-20 13:03:51 +00001965 finish_wait(sk_sleep(sk), &wait);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001966 return timeo;
1967}
1968
1969
1970/*
1971 * Generic send/receive buffer handlers
1972 */
1973
Herbert Xu4cc7f682009-02-04 16:55:54 -08001974struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len,
1975 unsigned long data_len, int noblock,
Eric Dumazet28d64272013-08-08 14:38:47 -07001976 int *errcode, int max_page_order)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001977{
Eric Dumazet2e4e4412014-09-17 04:49:49 -07001978 struct sk_buff *skb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001979 long timeo;
1980 int err;
1981
Linus Torvalds1da177e2005-04-16 15:20:36 -07001982 timeo = sock_sndtimeo(sk, noblock);
Eric Dumazet2e4e4412014-09-17 04:49:49 -07001983 for (;;) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001984 err = sock_error(sk);
1985 if (err != 0)
1986 goto failure;
1987
1988 err = -EPIPE;
1989 if (sk->sk_shutdown & SEND_SHUTDOWN)
1990 goto failure;
1991
Eric Dumazet2e4e4412014-09-17 04:49:49 -07001992 if (sk_wmem_alloc_get(sk) < sk->sk_sndbuf)
1993 break;
Eric Dumazet28d64272013-08-08 14:38:47 -07001994
Eric Dumazet9cd3e072015-11-29 20:03:10 -08001995 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
Eric Dumazet2e4e4412014-09-17 04:49:49 -07001996 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1997 err = -EAGAIN;
1998 if (!timeo)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001999 goto failure;
Eric Dumazet2e4e4412014-09-17 04:49:49 -07002000 if (signal_pending(current))
2001 goto interrupted;
2002 timeo = sock_wait_for_wmem(sk, timeo);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002003 }
Eric Dumazet2e4e4412014-09-17 04:49:49 -07002004 skb = alloc_skb_with_frags(header_len, data_len, max_page_order,
2005 errcode, sk->sk_allocation);
2006 if (skb)
2007 skb_set_owner_w(skb, sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002008 return skb;
2009
2010interrupted:
2011 err = sock_intr_errno(timeo);
2012failure:
2013 *errcode = err;
2014 return NULL;
2015}
Herbert Xu4cc7f682009-02-04 16:55:54 -08002016EXPORT_SYMBOL(sock_alloc_send_pskb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002017
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002018struct sk_buff *sock_alloc_send_skb(struct sock *sk, unsigned long size,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002019 int noblock, int *errcode)
2020{
Eric Dumazet28d64272013-08-08 14:38:47 -07002021 return sock_alloc_send_pskb(sk, size, 0, noblock, errcode, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002022}
Eric Dumazet2a915252009-05-27 11:30:05 +00002023EXPORT_SYMBOL(sock_alloc_send_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002024
Willem de Bruijn39771b12016-04-02 23:08:06 -04002025int __sock_cmsg_send(struct sock *sk, struct msghdr *msg, struct cmsghdr *cmsg,
2026 struct sockcm_cookie *sockc)
2027{
Soheil Hassas Yeganeh3dd17e62016-04-02 23:08:09 -04002028 u32 tsflags;
2029
Willem de Bruijn39771b12016-04-02 23:08:06 -04002030 switch (cmsg->cmsg_type) {
2031 case SO_MARK:
2032 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
2033 return -EPERM;
2034 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32)))
2035 return -EINVAL;
2036 sockc->mark = *(u32 *)CMSG_DATA(cmsg);
2037 break;
Soheil Hassas Yeganeh3dd17e62016-04-02 23:08:09 -04002038 case SO_TIMESTAMPING:
2039 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32)))
2040 return -EINVAL;
2041
2042 tsflags = *(u32 *)CMSG_DATA(cmsg);
2043 if (tsflags & ~SOF_TIMESTAMPING_TX_RECORD_MASK)
2044 return -EINVAL;
2045
2046 sockc->tsflags &= ~SOF_TIMESTAMPING_TX_RECORD_MASK;
2047 sockc->tsflags |= tsflags;
2048 break;
Soheil Hassas Yeganeh779f1ed2016-07-11 16:51:26 -04002049 /* SCM_RIGHTS and SCM_CREDENTIALS are semantically in SOL_UNIX. */
2050 case SCM_RIGHTS:
2051 case SCM_CREDENTIALS:
2052 break;
Willem de Bruijn39771b12016-04-02 23:08:06 -04002053 default:
2054 return -EINVAL;
2055 }
2056 return 0;
2057}
2058EXPORT_SYMBOL(__sock_cmsg_send);
2059
Edward Jeef28ea362015-10-08 14:56:48 -07002060int sock_cmsg_send(struct sock *sk, struct msghdr *msg,
2061 struct sockcm_cookie *sockc)
2062{
2063 struct cmsghdr *cmsg;
Willem de Bruijn39771b12016-04-02 23:08:06 -04002064 int ret;
Edward Jeef28ea362015-10-08 14:56:48 -07002065
2066 for_each_cmsghdr(cmsg, msg) {
2067 if (!CMSG_OK(msg, cmsg))
2068 return -EINVAL;
2069 if (cmsg->cmsg_level != SOL_SOCKET)
2070 continue;
Willem de Bruijn39771b12016-04-02 23:08:06 -04002071 ret = __sock_cmsg_send(sk, msg, cmsg, sockc);
2072 if (ret)
2073 return ret;
Edward Jeef28ea362015-10-08 14:56:48 -07002074 }
2075 return 0;
2076}
2077EXPORT_SYMBOL(sock_cmsg_send);
2078
Eric Dumazet06044752017-06-07 13:29:12 -07002079static void sk_enter_memory_pressure(struct sock *sk)
2080{
2081 if (!sk->sk_prot->enter_memory_pressure)
2082 return;
2083
2084 sk->sk_prot->enter_memory_pressure(sk);
2085}
2086
2087static void sk_leave_memory_pressure(struct sock *sk)
2088{
2089 if (sk->sk_prot->leave_memory_pressure) {
2090 sk->sk_prot->leave_memory_pressure(sk);
2091 } else {
2092 unsigned long *memory_pressure = sk->sk_prot->memory_pressure;
2093
2094 if (memory_pressure && *memory_pressure)
2095 *memory_pressure = 0;
2096 }
2097}
2098
Eric Dumazet5640f762012-09-23 23:04:42 +00002099/* On 32bit arches, an skb frag is limited to 2^15 */
2100#define SKB_FRAG_PAGE_ORDER get_order(32768)
2101
Eric Dumazet400dfd32013-10-17 16:27:07 -07002102/**
2103 * skb_page_frag_refill - check that a page_frag contains enough room
2104 * @sz: minimum size of the fragment we want to get
2105 * @pfrag: pointer to page_frag
Eric Dumazet82d5e2b2014-09-08 04:00:00 -07002106 * @gfp: priority for memory allocation
Eric Dumazet400dfd32013-10-17 16:27:07 -07002107 *
2108 * Note: While this allocator tries to use high order pages, there is
2109 * no guarantee that allocations succeed. Therefore, @sz MUST be
2110 * less or equal than PAGE_SIZE.
2111 */
Eric Dumazetd9b29382014-08-27 20:49:34 -07002112bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t gfp)
Eric Dumazet5640f762012-09-23 23:04:42 +00002113{
Eric Dumazet5640f762012-09-23 23:04:42 +00002114 if (pfrag->page) {
Joonsoo Kimfe896d12016-03-17 14:19:26 -07002115 if (page_ref_count(pfrag->page) == 1) {
Eric Dumazet5640f762012-09-23 23:04:42 +00002116 pfrag->offset = 0;
2117 return true;
2118 }
Eric Dumazet400dfd32013-10-17 16:27:07 -07002119 if (pfrag->offset + sz <= pfrag->size)
Eric Dumazet5640f762012-09-23 23:04:42 +00002120 return true;
2121 put_page(pfrag->page);
2122 }
2123
Eric Dumazetd9b29382014-08-27 20:49:34 -07002124 pfrag->offset = 0;
2125 if (SKB_FRAG_PAGE_ORDER) {
Mel Gormand0164ad2015-11-06 16:28:21 -08002126 /* Avoid direct reclaim but allow kswapd to wake */
2127 pfrag->page = alloc_pages((gfp & ~__GFP_DIRECT_RECLAIM) |
2128 __GFP_COMP | __GFP_NOWARN |
2129 __GFP_NORETRY,
Eric Dumazetd9b29382014-08-27 20:49:34 -07002130 SKB_FRAG_PAGE_ORDER);
Eric Dumazet5640f762012-09-23 23:04:42 +00002131 if (likely(pfrag->page)) {
Eric Dumazetd9b29382014-08-27 20:49:34 -07002132 pfrag->size = PAGE_SIZE << SKB_FRAG_PAGE_ORDER;
Eric Dumazet5640f762012-09-23 23:04:42 +00002133 return true;
2134 }
Eric Dumazetd9b29382014-08-27 20:49:34 -07002135 }
2136 pfrag->page = alloc_page(gfp);
2137 if (likely(pfrag->page)) {
2138 pfrag->size = PAGE_SIZE;
2139 return true;
2140 }
Eric Dumazet400dfd32013-10-17 16:27:07 -07002141 return false;
2142}
2143EXPORT_SYMBOL(skb_page_frag_refill);
2144
2145bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag)
2146{
2147 if (likely(skb_page_frag_refill(32U, pfrag, sk->sk_allocation)))
2148 return true;
2149
Eric Dumazet5640f762012-09-23 23:04:42 +00002150 sk_enter_memory_pressure(sk);
2151 sk_stream_moderate_sndbuf(sk);
2152 return false;
2153}
2154EXPORT_SYMBOL(sk_page_frag_refill);
2155
Linus Torvalds1da177e2005-04-16 15:20:36 -07002156static void __lock_sock(struct sock *sk)
Namhyung Kimf39234d2010-09-08 03:48:48 +00002157 __releases(&sk->sk_lock.slock)
2158 __acquires(&sk->sk_lock.slock)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002159{
2160 DEFINE_WAIT(wait);
2161
Stephen Hemmingere71a4782007-04-10 20:10:33 -07002162 for (;;) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002163 prepare_to_wait_exclusive(&sk->sk_lock.wq, &wait,
2164 TASK_UNINTERRUPTIBLE);
2165 spin_unlock_bh(&sk->sk_lock.slock);
2166 schedule();
2167 spin_lock_bh(&sk->sk_lock.slock);
Stephen Hemmingere71a4782007-04-10 20:10:33 -07002168 if (!sock_owned_by_user(sk))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002169 break;
2170 }
2171 finish_wait(&sk->sk_lock.wq, &wait);
2172}
2173
2174static void __release_sock(struct sock *sk)
Namhyung Kimf39234d2010-09-08 03:48:48 +00002175 __releases(&sk->sk_lock.slock)
2176 __acquires(&sk->sk_lock.slock)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002177{
Eric Dumazet5413d1b2016-04-29 14:16:52 -07002178 struct sk_buff *skb, *next;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002179
Eric Dumazet5413d1b2016-04-29 14:16:52 -07002180 while ((skb = sk->sk_backlog.head) != NULL) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002181 sk->sk_backlog.head = sk->sk_backlog.tail = NULL;
Eric Dumazet5413d1b2016-04-29 14:16:52 -07002182
2183 spin_unlock_bh(&sk->sk_lock.slock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002184
2185 do {
Eric Dumazet5413d1b2016-04-29 14:16:52 -07002186 next = skb->next;
Eric Dumazete4cbb022012-04-30 16:07:09 +00002187 prefetch(next);
Eric Dumazet7fee2262010-05-11 23:19:48 +00002188 WARN_ON_ONCE(skb_dst_is_noref(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002189 skb->next = NULL;
Peter Zijlstrac57943a2008-10-07 14:18:42 -07002190 sk_backlog_rcv(sk, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002191
Eric Dumazet5413d1b2016-04-29 14:16:52 -07002192 cond_resched();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002193
2194 skb = next;
2195 } while (skb != NULL);
2196
Eric Dumazet5413d1b2016-04-29 14:16:52 -07002197 spin_lock_bh(&sk->sk_lock.slock);
2198 }
Zhu Yi8eae9392010-03-04 18:01:40 +00002199
2200 /*
2201 * Doing the zeroing here guarantee we can not loop forever
2202 * while a wild producer attempts to flood us.
2203 */
2204 sk->sk_backlog.len = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002205}
2206
Eric Dumazetd41a69f2016-04-29 14:16:53 -07002207void __sk_flush_backlog(struct sock *sk)
2208{
2209 spin_lock_bh(&sk->sk_lock.slock);
2210 __release_sock(sk);
2211 spin_unlock_bh(&sk->sk_lock.slock);
2212}
2213
Linus Torvalds1da177e2005-04-16 15:20:36 -07002214/**
2215 * sk_wait_data - wait for data to arrive at sk_receive_queue
Pavel Pisa4dc3b162005-05-01 08:59:25 -07002216 * @sk: sock to wait on
2217 * @timeo: for how long
Sabrina Dubrocadfbafc92015-07-24 18:19:25 +02002218 * @skb: last skb seen on sk_receive_queue
Linus Torvalds1da177e2005-04-16 15:20:36 -07002219 *
2220 * Now socket state including sk->sk_err is changed only under lock,
2221 * hence we may omit checks after joining wait queue.
2222 * We check receive queue before schedule() only as optimization;
2223 * it is very likely that release_sock() added new data.
2224 */
Sabrina Dubrocadfbafc92015-07-24 18:19:25 +02002225int sk_wait_data(struct sock *sk, long *timeo, const struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002226{
WANG Congd9dc8b02016-11-11 10:20:50 -08002227 DEFINE_WAIT_FUNC(wait, woken_wake_function);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002228 int rc;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002229
WANG Congd9dc8b02016-11-11 10:20:50 -08002230 add_wait_queue(sk_sleep(sk), &wait);
Eric Dumazet9cd3e072015-11-29 20:03:10 -08002231 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
WANG Congd9dc8b02016-11-11 10:20:50 -08002232 rc = sk_wait_event(sk, timeo, skb_peek_tail(&sk->sk_receive_queue) != skb, &wait);
Eric Dumazet9cd3e072015-11-29 20:03:10 -08002233 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
WANG Congd9dc8b02016-11-11 10:20:50 -08002234 remove_wait_queue(sk_sleep(sk), &wait);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002235 return rc;
2236}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002237EXPORT_SYMBOL(sk_wait_data);
2238
Hideo Aoki3ab224b2007-12-31 00:11:19 -08002239/**
Paolo Abenif8c3bf02016-10-21 13:55:45 +02002240 * __sk_mem_raise_allocated - increase memory_allocated
Hideo Aoki3ab224b2007-12-31 00:11:19 -08002241 * @sk: socket
2242 * @size: memory size to allocate
Paolo Abenif8c3bf02016-10-21 13:55:45 +02002243 * @amt: pages to allocate
Hideo Aoki3ab224b2007-12-31 00:11:19 -08002244 * @kind: allocation type
2245 *
Paolo Abenif8c3bf02016-10-21 13:55:45 +02002246 * Similar to __sk_mem_schedule(), but does not update sk_forward_alloc
Hideo Aoki3ab224b2007-12-31 00:11:19 -08002247 */
Paolo Abenif8c3bf02016-10-21 13:55:45 +02002248int __sk_mem_raise_allocated(struct sock *sk, int size, int amt, int kind)
Hideo Aoki3ab224b2007-12-31 00:11:19 -08002249{
2250 struct proto *prot = sk->sk_prot;
Paolo Abenif8c3bf02016-10-21 13:55:45 +02002251 long allocated = sk_memory_allocated_add(sk, amt);
Johannes Weinere8056052016-01-14 15:21:14 -08002252
Johannes Weinerbaac50b2016-01-14 15:21:17 -08002253 if (mem_cgroup_sockets_enabled && sk->sk_memcg &&
2254 !mem_cgroup_charge_skmem(sk->sk_memcg, amt))
Johannes Weinere8056052016-01-14 15:21:14 -08002255 goto suppress_allocation;
Hideo Aoki3ab224b2007-12-31 00:11:19 -08002256
2257 /* Under limit. */
Johannes Weinere8056052016-01-14 15:21:14 -08002258 if (allocated <= sk_prot_mem_limits(sk, 0)) {
Glauber Costa180d8cd2011-12-11 21:47:02 +00002259 sk_leave_memory_pressure(sk);
Hideo Aoki3ab224b2007-12-31 00:11:19 -08002260 return 1;
2261 }
2262
Johannes Weinere8056052016-01-14 15:21:14 -08002263 /* Under pressure. */
2264 if (allocated > sk_prot_mem_limits(sk, 1))
Glauber Costa180d8cd2011-12-11 21:47:02 +00002265 sk_enter_memory_pressure(sk);
Hideo Aoki3ab224b2007-12-31 00:11:19 -08002266
Johannes Weinere8056052016-01-14 15:21:14 -08002267 /* Over hard limit. */
2268 if (allocated > sk_prot_mem_limits(sk, 2))
Hideo Aoki3ab224b2007-12-31 00:11:19 -08002269 goto suppress_allocation;
2270
2271 /* guarantee minimum buffer size under pressure */
2272 if (kind == SK_MEM_RECV) {
2273 if (atomic_read(&sk->sk_rmem_alloc) < prot->sysctl_rmem[0])
2274 return 1;
Glauber Costa180d8cd2011-12-11 21:47:02 +00002275
Hideo Aoki3ab224b2007-12-31 00:11:19 -08002276 } else { /* SK_MEM_SEND */
2277 if (sk->sk_type == SOCK_STREAM) {
2278 if (sk->sk_wmem_queued < prot->sysctl_wmem[0])
2279 return 1;
2280 } else if (atomic_read(&sk->sk_wmem_alloc) <
2281 prot->sysctl_wmem[0])
2282 return 1;
2283 }
2284
Glauber Costa180d8cd2011-12-11 21:47:02 +00002285 if (sk_has_memory_pressure(sk)) {
Eric Dumazet17483762008-11-25 21:16:35 -08002286 int alloc;
2287
Glauber Costa180d8cd2011-12-11 21:47:02 +00002288 if (!sk_under_memory_pressure(sk))
Eric Dumazet17483762008-11-25 21:16:35 -08002289 return 1;
Glauber Costa180d8cd2011-12-11 21:47:02 +00002290 alloc = sk_sockets_allocated_read_positive(sk);
2291 if (sk_prot_mem_limits(sk, 2) > alloc *
Hideo Aoki3ab224b2007-12-31 00:11:19 -08002292 sk_mem_pages(sk->sk_wmem_queued +
2293 atomic_read(&sk->sk_rmem_alloc) +
2294 sk->sk_forward_alloc))
2295 return 1;
2296 }
2297
2298suppress_allocation:
2299
2300 if (kind == SK_MEM_SEND && sk->sk_type == SOCK_STREAM) {
2301 sk_stream_moderate_sndbuf(sk);
2302
2303 /* Fail only if socket is _under_ its sndbuf.
2304 * In this case we cannot block, so that we have to fail.
2305 */
2306 if (sk->sk_wmem_queued + size >= sk->sk_sndbuf)
2307 return 1;
2308 }
2309
Satoru Moriya3847ce32011-06-17 12:00:03 +00002310 trace_sock_exceed_buf_limit(sk, prot, allocated);
2311
Glauber Costa0e90b312012-01-20 04:57:16 +00002312 sk_memory_allocated_sub(sk, amt);
Glauber Costa180d8cd2011-12-11 21:47:02 +00002313
Johannes Weinerbaac50b2016-01-14 15:21:17 -08002314 if (mem_cgroup_sockets_enabled && sk->sk_memcg)
2315 mem_cgroup_uncharge_skmem(sk->sk_memcg, amt);
Johannes Weinere8056052016-01-14 15:21:14 -08002316
Hideo Aoki3ab224b2007-12-31 00:11:19 -08002317 return 0;
2318}
Paolo Abenif8c3bf02016-10-21 13:55:45 +02002319EXPORT_SYMBOL(__sk_mem_raise_allocated);
2320
2321/**
2322 * __sk_mem_schedule - increase sk_forward_alloc and memory_allocated
2323 * @sk: socket
2324 * @size: memory size to allocate
2325 * @kind: allocation type
2326 *
2327 * If kind is SK_MEM_SEND, it means wmem allocation. Otherwise it means
2328 * rmem allocation. This function assumes that protocols which have
2329 * memory_pressure use sk_wmem_queued as write buffer accounting.
2330 */
2331int __sk_mem_schedule(struct sock *sk, int size, int kind)
2332{
2333 int ret, amt = sk_mem_pages(size);
2334
2335 sk->sk_forward_alloc += amt << SK_MEM_QUANTUM_SHIFT;
2336 ret = __sk_mem_raise_allocated(sk, size, amt, kind);
2337 if (!ret)
2338 sk->sk_forward_alloc -= amt << SK_MEM_QUANTUM_SHIFT;
2339 return ret;
2340}
Hideo Aoki3ab224b2007-12-31 00:11:19 -08002341EXPORT_SYMBOL(__sk_mem_schedule);
2342
2343/**
Paolo Abenif8c3bf02016-10-21 13:55:45 +02002344 * __sk_mem_reduce_allocated - reclaim memory_allocated
Hideo Aoki3ab224b2007-12-31 00:11:19 -08002345 * @sk: socket
Paolo Abenif8c3bf02016-10-21 13:55:45 +02002346 * @amount: number of quanta
2347 *
2348 * Similar to __sk_mem_reclaim(), but does not update sk_forward_alloc
Hideo Aoki3ab224b2007-12-31 00:11:19 -08002349 */
Paolo Abenif8c3bf02016-10-21 13:55:45 +02002350void __sk_mem_reduce_allocated(struct sock *sk, int amount)
Hideo Aoki3ab224b2007-12-31 00:11:19 -08002351{
Eric Dumazet1a24e042015-05-15 12:39:25 -07002352 sk_memory_allocated_sub(sk, amount);
Hideo Aoki3ab224b2007-12-31 00:11:19 -08002353
Johannes Weinerbaac50b2016-01-14 15:21:17 -08002354 if (mem_cgroup_sockets_enabled && sk->sk_memcg)
2355 mem_cgroup_uncharge_skmem(sk->sk_memcg, amount);
Johannes Weinere8056052016-01-14 15:21:14 -08002356
Glauber Costa180d8cd2011-12-11 21:47:02 +00002357 if (sk_under_memory_pressure(sk) &&
2358 (sk_memory_allocated(sk) < sk_prot_mem_limits(sk, 0)))
2359 sk_leave_memory_pressure(sk);
Hideo Aoki3ab224b2007-12-31 00:11:19 -08002360}
Paolo Abenif8c3bf02016-10-21 13:55:45 +02002361EXPORT_SYMBOL(__sk_mem_reduce_allocated);
2362
2363/**
2364 * __sk_mem_reclaim - reclaim sk_forward_alloc and memory_allocated
2365 * @sk: socket
2366 * @amount: number of bytes (rounded down to a SK_MEM_QUANTUM multiple)
2367 */
2368void __sk_mem_reclaim(struct sock *sk, int amount)
2369{
2370 amount >>= SK_MEM_QUANTUM_SHIFT;
2371 sk->sk_forward_alloc -= amount << SK_MEM_QUANTUM_SHIFT;
2372 __sk_mem_reduce_allocated(sk, amount);
2373}
Hideo Aoki3ab224b2007-12-31 00:11:19 -08002374EXPORT_SYMBOL(__sk_mem_reclaim);
2375
samanthakumar627d2d62016-04-05 12:41:16 -04002376int sk_set_peek_off(struct sock *sk, int val)
2377{
2378 if (val < 0)
2379 return -EINVAL;
2380
2381 sk->sk_peek_off = val;
2382 return 0;
2383}
2384EXPORT_SYMBOL_GPL(sk_set_peek_off);
Hideo Aoki3ab224b2007-12-31 00:11:19 -08002385
Linus Torvalds1da177e2005-04-16 15:20:36 -07002386/*
2387 * Set of default routines for initialising struct proto_ops when
2388 * the protocol does not support a particular function. In certain
2389 * cases where it makes no sense for a protocol to have a "do nothing"
2390 * function, some default processing is provided.
2391 */
2392
2393int sock_no_bind(struct socket *sock, struct sockaddr *saddr, int len)
2394{
2395 return -EOPNOTSUPP;
2396}
Eric Dumazet2a915252009-05-27 11:30:05 +00002397EXPORT_SYMBOL(sock_no_bind);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002398
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002399int sock_no_connect(struct socket *sock, struct sockaddr *saddr,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002400 int len, int flags)
2401{
2402 return -EOPNOTSUPP;
2403}
Eric Dumazet2a915252009-05-27 11:30:05 +00002404EXPORT_SYMBOL(sock_no_connect);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002405
2406int sock_no_socketpair(struct socket *sock1, struct socket *sock2)
2407{
2408 return -EOPNOTSUPP;
2409}
Eric Dumazet2a915252009-05-27 11:30:05 +00002410EXPORT_SYMBOL(sock_no_socketpair);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002411
David Howellscdfbabf2017-03-09 08:09:05 +00002412int sock_no_accept(struct socket *sock, struct socket *newsock, int flags,
2413 bool kern)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002414{
2415 return -EOPNOTSUPP;
2416}
Eric Dumazet2a915252009-05-27 11:30:05 +00002417EXPORT_SYMBOL(sock_no_accept);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002418
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002419int sock_no_getname(struct socket *sock, struct sockaddr *saddr,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002420 int *len, int peer)
2421{
2422 return -EOPNOTSUPP;
2423}
Eric Dumazet2a915252009-05-27 11:30:05 +00002424EXPORT_SYMBOL(sock_no_getname);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002425
Eric Dumazet2a915252009-05-27 11:30:05 +00002426unsigned int sock_no_poll(struct file *file, struct socket *sock, poll_table *pt)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002427{
2428 return 0;
2429}
Eric Dumazet2a915252009-05-27 11:30:05 +00002430EXPORT_SYMBOL(sock_no_poll);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002431
2432int sock_no_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg)
2433{
2434 return -EOPNOTSUPP;
2435}
Eric Dumazet2a915252009-05-27 11:30:05 +00002436EXPORT_SYMBOL(sock_no_ioctl);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002437
2438int sock_no_listen(struct socket *sock, int backlog)
2439{
2440 return -EOPNOTSUPP;
2441}
Eric Dumazet2a915252009-05-27 11:30:05 +00002442EXPORT_SYMBOL(sock_no_listen);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002443
2444int sock_no_shutdown(struct socket *sock, int how)
2445{
2446 return -EOPNOTSUPP;
2447}
Eric Dumazet2a915252009-05-27 11:30:05 +00002448EXPORT_SYMBOL(sock_no_shutdown);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002449
2450int sock_no_setsockopt(struct socket *sock, int level, int optname,
David S. Millerb7058842009-09-30 16:12:20 -07002451 char __user *optval, unsigned int optlen)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002452{
2453 return -EOPNOTSUPP;
2454}
Eric Dumazet2a915252009-05-27 11:30:05 +00002455EXPORT_SYMBOL(sock_no_setsockopt);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002456
2457int sock_no_getsockopt(struct socket *sock, int level, int optname,
2458 char __user *optval, int __user *optlen)
2459{
2460 return -EOPNOTSUPP;
2461}
Eric Dumazet2a915252009-05-27 11:30:05 +00002462EXPORT_SYMBOL(sock_no_getsockopt);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002463
Ying Xue1b784142015-03-02 15:37:48 +08002464int sock_no_sendmsg(struct socket *sock, struct msghdr *m, size_t len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002465{
2466 return -EOPNOTSUPP;
2467}
Eric Dumazet2a915252009-05-27 11:30:05 +00002468EXPORT_SYMBOL(sock_no_sendmsg);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002469
Ying Xue1b784142015-03-02 15:37:48 +08002470int sock_no_recvmsg(struct socket *sock, struct msghdr *m, size_t len,
2471 int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002472{
2473 return -EOPNOTSUPP;
2474}
Eric Dumazet2a915252009-05-27 11:30:05 +00002475EXPORT_SYMBOL(sock_no_recvmsg);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002476
2477int sock_no_mmap(struct file *file, struct socket *sock, struct vm_area_struct *vma)
2478{
2479 /* Mirror missing mmap method error code */
2480 return -ENODEV;
2481}
Eric Dumazet2a915252009-05-27 11:30:05 +00002482EXPORT_SYMBOL(sock_no_mmap);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002483
2484ssize_t sock_no_sendpage(struct socket *sock, struct page *page, int offset, size_t size, int flags)
2485{
2486 ssize_t res;
2487 struct msghdr msg = {.msg_flags = flags};
2488 struct kvec iov;
2489 char *kaddr = kmap(page);
2490 iov.iov_base = kaddr + offset;
2491 iov.iov_len = size;
2492 res = kernel_sendmsg(sock, &msg, &iov, 1, size);
2493 kunmap(page);
2494 return res;
2495}
Eric Dumazet2a915252009-05-27 11:30:05 +00002496EXPORT_SYMBOL(sock_no_sendpage);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002497
2498/*
2499 * Default Socket Callbacks
2500 */
2501
2502static void sock_def_wakeup(struct sock *sk)
2503{
Eric Dumazet43815482010-04-29 11:01:49 +00002504 struct socket_wq *wq;
2505
2506 rcu_read_lock();
2507 wq = rcu_dereference(sk->sk_wq);
Herbert Xu1ce0bf52015-11-26 13:55:39 +08002508 if (skwq_has_sleeper(wq))
Eric Dumazet43815482010-04-29 11:01:49 +00002509 wake_up_interruptible_all(&wq->wait);
2510 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002511}
2512
2513static void sock_def_error_report(struct sock *sk)
2514{
Eric Dumazet43815482010-04-29 11:01:49 +00002515 struct socket_wq *wq;
2516
2517 rcu_read_lock();
2518 wq = rcu_dereference(sk->sk_wq);
Herbert Xu1ce0bf52015-11-26 13:55:39 +08002519 if (skwq_has_sleeper(wq))
Eric Dumazet43815482010-04-29 11:01:49 +00002520 wake_up_interruptible_poll(&wq->wait, POLLERR);
Pavel Emelyanov8d8ad9d2007-11-26 20:10:50 +08002521 sk_wake_async(sk, SOCK_WAKE_IO, POLL_ERR);
Eric Dumazet43815482010-04-29 11:01:49 +00002522 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002523}
2524
David S. Miller676d2362014-04-11 16:15:36 -04002525static void sock_def_readable(struct sock *sk)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002526{
Eric Dumazet43815482010-04-29 11:01:49 +00002527 struct socket_wq *wq;
2528
2529 rcu_read_lock();
2530 wq = rcu_dereference(sk->sk_wq);
Herbert Xu1ce0bf52015-11-26 13:55:39 +08002531 if (skwq_has_sleeper(wq))
Eric Dumazet2c6607c2011-01-06 10:54:29 -08002532 wake_up_interruptible_sync_poll(&wq->wait, POLLIN | POLLPRI |
Davide Libenzi37e55402009-03-31 15:24:21 -07002533 POLLRDNORM | POLLRDBAND);
Pavel Emelyanov8d8ad9d2007-11-26 20:10:50 +08002534 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
Eric Dumazet43815482010-04-29 11:01:49 +00002535 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002536}
2537
2538static void sock_def_write_space(struct sock *sk)
2539{
Eric Dumazet43815482010-04-29 11:01:49 +00002540 struct socket_wq *wq;
2541
2542 rcu_read_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002543
2544 /* Do not wake up a writer until he can make "significant"
2545 * progress. --DaveM
2546 */
Stephen Hemmingere71a4782007-04-10 20:10:33 -07002547 if ((atomic_read(&sk->sk_wmem_alloc) << 1) <= sk->sk_sndbuf) {
Eric Dumazet43815482010-04-29 11:01:49 +00002548 wq = rcu_dereference(sk->sk_wq);
Herbert Xu1ce0bf52015-11-26 13:55:39 +08002549 if (skwq_has_sleeper(wq))
Eric Dumazet43815482010-04-29 11:01:49 +00002550 wake_up_interruptible_sync_poll(&wq->wait, POLLOUT |
Davide Libenzi37e55402009-03-31 15:24:21 -07002551 POLLWRNORM | POLLWRBAND);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002552
2553 /* Should agree with poll, otherwise some programs break */
2554 if (sock_writeable(sk))
Pavel Emelyanov8d8ad9d2007-11-26 20:10:50 +08002555 sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002556 }
2557
Eric Dumazet43815482010-04-29 11:01:49 +00002558 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002559}
2560
2561static void sock_def_destruct(struct sock *sk)
2562{
Linus Torvalds1da177e2005-04-16 15:20:36 -07002563}
2564
2565void sk_send_sigurg(struct sock *sk)
2566{
2567 if (sk->sk_socket && sk->sk_socket->file)
2568 if (send_sigurg(&sk->sk_socket->file->f_owner))
Pavel Emelyanov8d8ad9d2007-11-26 20:10:50 +08002569 sk_wake_async(sk, SOCK_WAKE_URG, POLL_PRI);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002570}
Eric Dumazet2a915252009-05-27 11:30:05 +00002571EXPORT_SYMBOL(sk_send_sigurg);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002572
2573void sk_reset_timer(struct sock *sk, struct timer_list* timer,
2574 unsigned long expires)
2575{
2576 if (!mod_timer(timer, expires))
2577 sock_hold(sk);
2578}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002579EXPORT_SYMBOL(sk_reset_timer);
2580
2581void sk_stop_timer(struct sock *sk, struct timer_list* timer)
2582{
Ying Xue25cc4ae2013-02-03 20:32:57 +00002583 if (del_timer(timer))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002584 __sock_put(sk);
2585}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002586EXPORT_SYMBOL(sk_stop_timer);
2587
2588void sock_init_data(struct socket *sock, struct sock *sk)
2589{
Paolo Abeni581319c2017-03-09 13:54:08 +01002590 sk_init_common(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002591 sk->sk_send_head = NULL;
2592
2593 init_timer(&sk->sk_timer);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002594
Linus Torvalds1da177e2005-04-16 15:20:36 -07002595 sk->sk_allocation = GFP_KERNEL;
2596 sk->sk_rcvbuf = sysctl_rmem_default;
2597 sk->sk_sndbuf = sysctl_wmem_default;
2598 sk->sk_state = TCP_CLOSE;
David S. Miller972692e2008-06-17 22:41:38 -07002599 sk_set_socket(sk, sock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002600
2601 sock_set_flag(sk, SOCK_ZAPPED);
2602
Stephen Hemmingere71a4782007-04-10 20:10:33 -07002603 if (sock) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002604 sk->sk_type = sock->type;
Eric Dumazet43815482010-04-29 11:01:49 +00002605 sk->sk_wq = sock->wq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002606 sock->sk = sk;
Lorenzo Colitti86741ec2016-11-04 02:23:41 +09002607 sk->sk_uid = SOCK_INODE(sock)->i_uid;
2608 } else {
Eric Dumazet43815482010-04-29 11:01:49 +00002609 sk->sk_wq = NULL;
Lorenzo Colitti86741ec2016-11-04 02:23:41 +09002610 sk->sk_uid = make_kuid(sock_net(sk)->user_ns, 0);
2611 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002612
Linus Torvalds1da177e2005-04-16 15:20:36 -07002613 rwlock_init(&sk->sk_callback_lock);
David Howellscdfbabf2017-03-09 08:09:05 +00002614 if (sk->sk_kern_sock)
2615 lockdep_set_class_and_name(
2616 &sk->sk_callback_lock,
2617 af_kern_callback_keys + sk->sk_family,
2618 af_family_kern_clock_key_strings[sk->sk_family]);
2619 else
2620 lockdep_set_class_and_name(
2621 &sk->sk_callback_lock,
Peter Zijlstra443aef0e2007-07-19 01:49:00 -07002622 af_callback_keys + sk->sk_family,
2623 af_family_clock_key_strings[sk->sk_family]);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002624
2625 sk->sk_state_change = sock_def_wakeup;
2626 sk->sk_data_ready = sock_def_readable;
2627 sk->sk_write_space = sock_def_write_space;
2628 sk->sk_error_report = sock_def_error_report;
2629 sk->sk_destruct = sock_def_destruct;
2630
Eric Dumazet5640f762012-09-23 23:04:42 +00002631 sk->sk_frag.page = NULL;
2632 sk->sk_frag.offset = 0;
Pavel Emelyanovef64a542012-02-21 07:31:34 +00002633 sk->sk_peek_off = -1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002634
Eric W. Biederman109f6e32010-06-13 03:30:14 +00002635 sk->sk_peer_pid = NULL;
2636 sk->sk_peer_cred = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002637 sk->sk_write_pending = 0;
2638 sk->sk_rcvlowat = 1;
2639 sk->sk_rcvtimeo = MAX_SCHEDULE_TIMEOUT;
2640 sk->sk_sndtimeo = MAX_SCHEDULE_TIMEOUT;
2641
Paolo Abeni6c7c98b2017-03-30 14:03:06 +02002642 sk->sk_stamp = SK_DEFAULT_STAMP;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002643
Cong Wange0d10952013-08-01 11:10:25 +08002644#ifdef CONFIG_NET_RX_BUSY_POLL
Eliezer Tamir06021292013-06-10 11:39:50 +03002645 sk->sk_napi_id = 0;
Eliezer Tamir64b0dc52013-07-10 17:13:36 +03002646 sk->sk_ll_usec = sysctl_net_busy_read;
Eliezer Tamir06021292013-06-10 11:39:50 +03002647#endif
2648
Eric Dumazet62748f32013-09-24 08:20:52 -07002649 sk->sk_max_pacing_rate = ~0U;
Eric Dumazet7eec4172013-10-08 15:16:00 -07002650 sk->sk_pacing_rate = ~0U;
Eric Dumazet70da2682015-10-08 19:33:21 -07002651 sk->sk_incoming_cpu = -1;
Eric Dumazet4dc6dc72009-07-15 23:13:10 +00002652 /*
2653 * Before updating sk_refcnt, we must commit prior changes to memory
2654 * (Documentation/RCU/rculist_nulls.txt for details)
2655 */
2656 smp_wmb();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002657 atomic_set(&sk->sk_refcnt, 1);
Wang Chen33c732c2007-11-13 20:30:01 -08002658 atomic_set(&sk->sk_drops, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002659}
Eric Dumazet2a915252009-05-27 11:30:05 +00002660EXPORT_SYMBOL(sock_init_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002661
Harvey Harrisonb5606c22008-02-13 15:03:16 -08002662void lock_sock_nested(struct sock *sk, int subclass)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002663{
2664 might_sleep();
Ingo Molnara5b5bb92006-07-03 00:25:35 -07002665 spin_lock_bh(&sk->sk_lock.slock);
John Heffnerd2e91172007-09-12 10:44:19 +02002666 if (sk->sk_lock.owned)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002667 __lock_sock(sk);
John Heffnerd2e91172007-09-12 10:44:19 +02002668 sk->sk_lock.owned = 1;
Ingo Molnara5b5bb92006-07-03 00:25:35 -07002669 spin_unlock(&sk->sk_lock.slock);
2670 /*
2671 * The sk_lock has mutex_lock() semantics here:
2672 */
Peter Zijlstrafcc70d52006-11-08 22:44:35 -08002673 mutex_acquire(&sk->sk_lock.dep_map, subclass, 0, _RET_IP_);
Ingo Molnara5b5bb92006-07-03 00:25:35 -07002674 local_bh_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002675}
Peter Zijlstrafcc70d52006-11-08 22:44:35 -08002676EXPORT_SYMBOL(lock_sock_nested);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002677
Harvey Harrisonb5606c22008-02-13 15:03:16 -08002678void release_sock(struct sock *sk)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002679{
Ingo Molnara5b5bb92006-07-03 00:25:35 -07002680 spin_lock_bh(&sk->sk_lock.slock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002681 if (sk->sk_backlog.tail)
2682 __release_sock(sk);
Eric Dumazet46d3cea2012-07-11 05:50:31 +00002683
Eric Dumazetc3f9b012014-03-10 09:50:11 -07002684 /* Warning : release_cb() might need to release sk ownership,
2685 * ie call sock_release_ownership(sk) before us.
2686 */
Eric Dumazet46d3cea2012-07-11 05:50:31 +00002687 if (sk->sk_prot->release_cb)
2688 sk->sk_prot->release_cb(sk);
2689
Eric Dumazetc3f9b012014-03-10 09:50:11 -07002690 sock_release_ownership(sk);
Ingo Molnara5b5bb92006-07-03 00:25:35 -07002691 if (waitqueue_active(&sk->sk_lock.wq))
2692 wake_up(&sk->sk_lock.wq);
2693 spin_unlock_bh(&sk->sk_lock.slock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002694}
2695EXPORT_SYMBOL(release_sock);
2696
Eric Dumazet8a74ad62010-05-26 19:20:18 +00002697/**
2698 * lock_sock_fast - fast version of lock_sock
2699 * @sk: socket
2700 *
2701 * This version should be used for very small section, where process wont block
2702 * return false if fast path is taken
2703 * sk_lock.slock locked, owned = 0, BH disabled
2704 * return true if slow path is taken
2705 * sk_lock.slock unlocked, owned = 1, BH enabled
2706 */
2707bool lock_sock_fast(struct sock *sk)
2708{
2709 might_sleep();
2710 spin_lock_bh(&sk->sk_lock.slock);
2711
2712 if (!sk->sk_lock.owned)
2713 /*
2714 * Note : We must disable BH
2715 */
2716 return false;
2717
2718 __lock_sock(sk);
2719 sk->sk_lock.owned = 1;
2720 spin_unlock(&sk->sk_lock.slock);
2721 /*
2722 * The sk_lock has mutex_lock() semantics here:
2723 */
2724 mutex_acquire(&sk->sk_lock.dep_map, 0, 0, _RET_IP_);
2725 local_bh_enable();
2726 return true;
2727}
2728EXPORT_SYMBOL(lock_sock_fast);
2729
Linus Torvalds1da177e2005-04-16 15:20:36 -07002730int sock_get_timestamp(struct sock *sk, struct timeval __user *userstamp)
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002731{
Eric Dumazetb7aa0bf2007-04-19 16:16:32 -07002732 struct timeval tv;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002733 if (!sock_flag(sk, SOCK_TIMESTAMP))
Patrick Ohly20d49472009-02-12 05:03:38 +00002734 sock_enable_timestamp(sk, SOCK_TIMESTAMP);
Eric Dumazetb7aa0bf2007-04-19 16:16:32 -07002735 tv = ktime_to_timeval(sk->sk_stamp);
2736 if (tv.tv_sec == -1)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002737 return -ENOENT;
Eric Dumazetb7aa0bf2007-04-19 16:16:32 -07002738 if (tv.tv_sec == 0) {
2739 sk->sk_stamp = ktime_get_real();
2740 tv = ktime_to_timeval(sk->sk_stamp);
2741 }
2742 return copy_to_user(userstamp, &tv, sizeof(tv)) ? -EFAULT : 0;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002743}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002744EXPORT_SYMBOL(sock_get_timestamp);
2745
Eric Dumazetae40eb12007-03-18 17:33:16 -07002746int sock_get_timestampns(struct sock *sk, struct timespec __user *userstamp)
2747{
2748 struct timespec ts;
2749 if (!sock_flag(sk, SOCK_TIMESTAMP))
Patrick Ohly20d49472009-02-12 05:03:38 +00002750 sock_enable_timestamp(sk, SOCK_TIMESTAMP);
Eric Dumazetae40eb12007-03-18 17:33:16 -07002751 ts = ktime_to_timespec(sk->sk_stamp);
2752 if (ts.tv_sec == -1)
2753 return -ENOENT;
2754 if (ts.tv_sec == 0) {
2755 sk->sk_stamp = ktime_get_real();
2756 ts = ktime_to_timespec(sk->sk_stamp);
2757 }
2758 return copy_to_user(userstamp, &ts, sizeof(ts)) ? -EFAULT : 0;
2759}
2760EXPORT_SYMBOL(sock_get_timestampns);
2761
Patrick Ohly20d49472009-02-12 05:03:38 +00002762void sock_enable_timestamp(struct sock *sk, int flag)
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002763{
Patrick Ohly20d49472009-02-12 05:03:38 +00002764 if (!sock_flag(sk, flag)) {
Eric Dumazet08e29af2011-11-28 12:04:18 +00002765 unsigned long previous_flags = sk->sk_flags;
2766
Patrick Ohly20d49472009-02-12 05:03:38 +00002767 sock_set_flag(sk, flag);
2768 /*
2769 * we just set one of the two flags which require net
2770 * time stamping, but time stamping might have been on
2771 * already because of the other one
2772 */
Hannes Frederic Sowa080a2702015-10-26 13:51:37 +01002773 if (sock_needs_netstamp(sk) &&
2774 !(previous_flags & SK_FLAGS_TIMESTAMP))
Patrick Ohly20d49472009-02-12 05:03:38 +00002775 net_enable_timestamp();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002776 }
2777}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002778
Richard Cochrancb820f82013-07-19 19:40:09 +02002779int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len,
2780 int level, int type)
2781{
2782 struct sock_exterr_skb *serr;
Willem de Bruijn364a9e92014-08-31 21:30:27 -04002783 struct sk_buff *skb;
Richard Cochrancb820f82013-07-19 19:40:09 +02002784 int copied, err;
2785
2786 err = -EAGAIN;
Willem de Bruijn364a9e92014-08-31 21:30:27 -04002787 skb = sock_dequeue_err_skb(sk);
Richard Cochrancb820f82013-07-19 19:40:09 +02002788 if (skb == NULL)
2789 goto out;
2790
2791 copied = skb->len;
2792 if (copied > len) {
2793 msg->msg_flags |= MSG_TRUNC;
2794 copied = len;
2795 }
David S. Miller51f3d022014-11-05 16:46:40 -05002796 err = skb_copy_datagram_msg(skb, 0, msg, copied);
Richard Cochrancb820f82013-07-19 19:40:09 +02002797 if (err)
2798 goto out_free_skb;
2799
2800 sock_recv_timestamp(msg, sk, skb);
2801
2802 serr = SKB_EXT_ERR(skb);
2803 put_cmsg(msg, level, type, sizeof(serr->ee), &serr->ee);
2804
2805 msg->msg_flags |= MSG_ERRQUEUE;
2806 err = copied;
2807
Richard Cochrancb820f82013-07-19 19:40:09 +02002808out_free_skb:
2809 kfree_skb(skb);
2810out:
2811 return err;
2812}
2813EXPORT_SYMBOL(sock_recv_errqueue);
2814
Linus Torvalds1da177e2005-04-16 15:20:36 -07002815/*
2816 * Get a socket option on an socket.
2817 *
2818 * FIX: POSIX 1003.1g is very ambiguous here. It states that
2819 * asynchronous errors should be reported by getsockopt. We assume
2820 * this means if you specify SO_ERROR (otherwise whats the point of it).
2821 */
2822int sock_common_getsockopt(struct socket *sock, int level, int optname,
2823 char __user *optval, int __user *optlen)
2824{
2825 struct sock *sk = sock->sk;
2826
2827 return sk->sk_prot->getsockopt(sk, level, optname, optval, optlen);
2828}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002829EXPORT_SYMBOL(sock_common_getsockopt);
2830
Dmitry Mishin3fdadf72006-03-20 22:45:21 -08002831#ifdef CONFIG_COMPAT
Arnaldo Carvalho de Melo543d9cf2006-03-20 22:48:35 -08002832int compat_sock_common_getsockopt(struct socket *sock, int level, int optname,
2833 char __user *optval, int __user *optlen)
Dmitry Mishin3fdadf72006-03-20 22:45:21 -08002834{
2835 struct sock *sk = sock->sk;
2836
Johannes Berg1e51f952007-03-06 13:44:06 -08002837 if (sk->sk_prot->compat_getsockopt != NULL)
Arnaldo Carvalho de Melo543d9cf2006-03-20 22:48:35 -08002838 return sk->sk_prot->compat_getsockopt(sk, level, optname,
2839 optval, optlen);
Dmitry Mishin3fdadf72006-03-20 22:45:21 -08002840 return sk->sk_prot->getsockopt(sk, level, optname, optval, optlen);
2841}
2842EXPORT_SYMBOL(compat_sock_common_getsockopt);
2843#endif
2844
Ying Xue1b784142015-03-02 15:37:48 +08002845int sock_common_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
2846 int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002847{
2848 struct sock *sk = sock->sk;
2849 int addr_len = 0;
2850 int err;
2851
Ying Xue1b784142015-03-02 15:37:48 +08002852 err = sk->sk_prot->recvmsg(sk, msg, size, flags & MSG_DONTWAIT,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002853 flags & ~MSG_DONTWAIT, &addr_len);
2854 if (err >= 0)
2855 msg->msg_namelen = addr_len;
2856 return err;
2857}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002858EXPORT_SYMBOL(sock_common_recvmsg);
2859
2860/*
2861 * Set socket options on an inet socket.
2862 */
2863int sock_common_setsockopt(struct socket *sock, int level, int optname,
David S. Millerb7058842009-09-30 16:12:20 -07002864 char __user *optval, unsigned int optlen)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002865{
2866 struct sock *sk = sock->sk;
2867
2868 return sk->sk_prot->setsockopt(sk, level, optname, optval, optlen);
2869}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002870EXPORT_SYMBOL(sock_common_setsockopt);
2871
Dmitry Mishin3fdadf72006-03-20 22:45:21 -08002872#ifdef CONFIG_COMPAT
Arnaldo Carvalho de Melo543d9cf2006-03-20 22:48:35 -08002873int compat_sock_common_setsockopt(struct socket *sock, int level, int optname,
David S. Millerb7058842009-09-30 16:12:20 -07002874 char __user *optval, unsigned int optlen)
Dmitry Mishin3fdadf72006-03-20 22:45:21 -08002875{
2876 struct sock *sk = sock->sk;
2877
Arnaldo Carvalho de Melo543d9cf2006-03-20 22:48:35 -08002878 if (sk->sk_prot->compat_setsockopt != NULL)
2879 return sk->sk_prot->compat_setsockopt(sk, level, optname,
2880 optval, optlen);
Dmitry Mishin3fdadf72006-03-20 22:45:21 -08002881 return sk->sk_prot->setsockopt(sk, level, optname, optval, optlen);
2882}
2883EXPORT_SYMBOL(compat_sock_common_setsockopt);
2884#endif
2885
Linus Torvalds1da177e2005-04-16 15:20:36 -07002886void sk_common_release(struct sock *sk)
2887{
2888 if (sk->sk_prot->destroy)
2889 sk->sk_prot->destroy(sk);
2890
2891 /*
2892 * Observation: when sock_common_release is called, processes have
2893 * no access to socket. But net still has.
2894 * Step one, detach it from networking:
2895 *
2896 * A. Remove from hash tables.
2897 */
2898
2899 sk->sk_prot->unhash(sk);
2900
2901 /*
2902 * In this point socket cannot receive new packets, but it is possible
2903 * that some packets are in flight because some CPU runs receiver and
2904 * did hash table lookup before we unhashed socket. They will achieve
2905 * receive queue and will be purged by socket destructor.
2906 *
2907 * Also we still have packets pending on receive queue and probably,
2908 * our own packets waiting in device queues. sock_destroy will drain
2909 * receive queue, but transmitted packets will delay socket destruction
2910 * until the last reference will be released.
2911 */
2912
2913 sock_orphan(sk);
2914
2915 xfrm_sk_free_policy(sk);
2916
Arnaldo Carvalho de Meloe6848972005-08-09 19:45:38 -07002917 sk_refcnt_debug_release(sk);
Eric Dumazet5640f762012-09-23 23:04:42 +00002918
Linus Torvalds1da177e2005-04-16 15:20:36 -07002919 sock_put(sk);
2920}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002921EXPORT_SYMBOL(sk_common_release);
2922
Josh Hunta2d133b2017-03-20 15:22:03 -04002923void sk_get_meminfo(const struct sock *sk, u32 *mem)
2924{
2925 memset(mem, 0, sizeof(*mem) * SK_MEMINFO_VARS);
2926
2927 mem[SK_MEMINFO_RMEM_ALLOC] = sk_rmem_alloc_get(sk);
2928 mem[SK_MEMINFO_RCVBUF] = sk->sk_rcvbuf;
2929 mem[SK_MEMINFO_WMEM_ALLOC] = sk_wmem_alloc_get(sk);
2930 mem[SK_MEMINFO_SNDBUF] = sk->sk_sndbuf;
2931 mem[SK_MEMINFO_FWD_ALLOC] = sk->sk_forward_alloc;
2932 mem[SK_MEMINFO_WMEM_QUEUED] = sk->sk_wmem_queued;
2933 mem[SK_MEMINFO_OPTMEM] = atomic_read(&sk->sk_omem_alloc);
2934 mem[SK_MEMINFO_BACKLOG] = sk->sk_backlog.len;
2935 mem[SK_MEMINFO_DROPS] = atomic_read(&sk->sk_drops);
2936}
2937
Pavel Emelyanov13ff3d62008-03-28 16:38:17 -07002938#ifdef CONFIG_PROC_FS
2939#define PROTO_INUSE_NR 64 /* should be enough for the first time */
Pavel Emelyanov1338d462008-03-28 16:38:43 -07002940struct prot_inuse {
2941 int val[PROTO_INUSE_NR];
2942};
Pavel Emelyanov13ff3d62008-03-28 16:38:17 -07002943
2944static DECLARE_BITMAP(proto_inuse_idx, PROTO_INUSE_NR);
Pavel Emelyanov70ee1152008-03-31 19:42:16 -07002945
2946#ifdef CONFIG_NET_NS
2947void sock_prot_inuse_add(struct net *net, struct proto *prot, int val)
2948{
Eric Dumazetd6d9ca02010-07-19 10:48:49 +00002949 __this_cpu_add(net->core.inuse->val[prot->inuse_idx], val);
Pavel Emelyanov70ee1152008-03-31 19:42:16 -07002950}
2951EXPORT_SYMBOL_GPL(sock_prot_inuse_add);
2952
2953int sock_prot_inuse_get(struct net *net, struct proto *prot)
2954{
2955 int cpu, idx = prot->inuse_idx;
2956 int res = 0;
2957
2958 for_each_possible_cpu(cpu)
2959 res += per_cpu_ptr(net->core.inuse, cpu)->val[idx];
2960
2961 return res >= 0 ? res : 0;
2962}
2963EXPORT_SYMBOL_GPL(sock_prot_inuse_get);
2964
Alexey Dobriyan2c8c1e72010-01-17 03:35:32 +00002965static int __net_init sock_inuse_init_net(struct net *net)
Pavel Emelyanov70ee1152008-03-31 19:42:16 -07002966{
2967 net->core.inuse = alloc_percpu(struct prot_inuse);
2968 return net->core.inuse ? 0 : -ENOMEM;
2969}
2970
Alexey Dobriyan2c8c1e72010-01-17 03:35:32 +00002971static void __net_exit sock_inuse_exit_net(struct net *net)
Pavel Emelyanov70ee1152008-03-31 19:42:16 -07002972{
2973 free_percpu(net->core.inuse);
2974}
2975
2976static struct pernet_operations net_inuse_ops = {
2977 .init = sock_inuse_init_net,
2978 .exit = sock_inuse_exit_net,
2979};
2980
2981static __init int net_inuse_init(void)
2982{
2983 if (register_pernet_subsys(&net_inuse_ops))
2984 panic("Cannot initialize net inuse counters");
2985
2986 return 0;
2987}
2988
2989core_initcall(net_inuse_init);
2990#else
Pavel Emelyanov1338d462008-03-28 16:38:43 -07002991static DEFINE_PER_CPU(struct prot_inuse, prot_inuse);
2992
Pavel Emelyanovc29a0bc2008-03-31 19:41:46 -07002993void sock_prot_inuse_add(struct net *net, struct proto *prot, int val)
Pavel Emelyanov1338d462008-03-28 16:38:43 -07002994{
Eric Dumazetd6d9ca02010-07-19 10:48:49 +00002995 __this_cpu_add(prot_inuse.val[prot->inuse_idx], val);
Pavel Emelyanov1338d462008-03-28 16:38:43 -07002996}
2997EXPORT_SYMBOL_GPL(sock_prot_inuse_add);
2998
Pavel Emelyanovc29a0bc2008-03-31 19:41:46 -07002999int sock_prot_inuse_get(struct net *net, struct proto *prot)
Pavel Emelyanov1338d462008-03-28 16:38:43 -07003000{
3001 int cpu, idx = prot->inuse_idx;
3002 int res = 0;
3003
3004 for_each_possible_cpu(cpu)
3005 res += per_cpu(prot_inuse, cpu).val[idx];
3006
3007 return res >= 0 ? res : 0;
3008}
3009EXPORT_SYMBOL_GPL(sock_prot_inuse_get);
Pavel Emelyanov70ee1152008-03-31 19:42:16 -07003010#endif
Pavel Emelyanov13ff3d62008-03-28 16:38:17 -07003011
3012static void assign_proto_idx(struct proto *prot)
3013{
3014 prot->inuse_idx = find_first_zero_bit(proto_inuse_idx, PROTO_INUSE_NR);
3015
3016 if (unlikely(prot->inuse_idx == PROTO_INUSE_NR - 1)) {
Joe Perchese005d192012-05-16 19:58:40 +00003017 pr_err("PROTO_INUSE_NR exhausted\n");
Pavel Emelyanov13ff3d62008-03-28 16:38:17 -07003018 return;
3019 }
3020
3021 set_bit(prot->inuse_idx, proto_inuse_idx);
3022}
3023
3024static void release_proto_idx(struct proto *prot)
3025{
3026 if (prot->inuse_idx != PROTO_INUSE_NR - 1)
3027 clear_bit(prot->inuse_idx, proto_inuse_idx);
3028}
3029#else
3030static inline void assign_proto_idx(struct proto *prot)
3031{
3032}
3033
3034static inline void release_proto_idx(struct proto *prot)
3035{
3036}
3037#endif
3038
Eric Dumazet0159dfd2015-03-12 16:44:07 -07003039static void req_prot_cleanup(struct request_sock_ops *rsk_prot)
3040{
3041 if (!rsk_prot)
3042 return;
3043 kfree(rsk_prot->slab_name);
3044 rsk_prot->slab_name = NULL;
Julia Lawalladf78ed2015-09-13 14:15:18 +02003045 kmem_cache_destroy(rsk_prot->slab);
3046 rsk_prot->slab = NULL;
Eric Dumazet0159dfd2015-03-12 16:44:07 -07003047}
3048
3049static int req_prot_init(const struct proto *prot)
3050{
3051 struct request_sock_ops *rsk_prot = prot->rsk_prot;
3052
3053 if (!rsk_prot)
3054 return 0;
3055
3056 rsk_prot->slab_name = kasprintf(GFP_KERNEL, "request_sock_%s",
3057 prot->name);
3058 if (!rsk_prot->slab_name)
3059 return -ENOMEM;
3060
3061 rsk_prot->slab = kmem_cache_create(rsk_prot->slab_name,
3062 rsk_prot->obj_size, 0,
Eric Dumazete96f78a2015-10-03 06:27:28 -07003063 prot->slab_flags, NULL);
Eric Dumazet0159dfd2015-03-12 16:44:07 -07003064
3065 if (!rsk_prot->slab) {
3066 pr_crit("%s: Can't create request sock SLAB cache!\n",
3067 prot->name);
3068 return -ENOMEM;
3069 }
3070 return 0;
3071}
3072
Linus Torvalds1da177e2005-04-16 15:20:36 -07003073int proto_register(struct proto *prot, int alloc_slab)
3074{
Linus Torvalds1da177e2005-04-16 15:20:36 -07003075 if (alloc_slab) {
3076 prot->slab = kmem_cache_create(prot->name, prot->obj_size, 0,
Eric Dumazet271b72c2008-10-29 02:11:14 -07003077 SLAB_HWCACHE_ALIGN | prot->slab_flags,
3078 NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003079
3080 if (prot->slab == NULL) {
Joe Perchese005d192012-05-16 19:58:40 +00003081 pr_crit("%s: Can't create sock SLAB cache!\n",
3082 prot->name);
Pavel Emelyanov60e76632008-03-28 16:39:10 -07003083 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003084 }
Arnaldo Carvalho de Melo2e6599c2005-06-18 22:46:52 -07003085
Eric Dumazet0159dfd2015-03-12 16:44:07 -07003086 if (req_prot_init(prot))
3087 goto out_free_request_sock_slab;
Arnaldo Carvalho de Melo8feaf0c02005-08-09 20:09:30 -07003088
Arnaldo Carvalho de Melo6d6ee432005-12-13 23:25:19 -08003089 if (prot->twsk_prot != NULL) {
Alexey Dobriyanfaf23422010-02-17 09:34:12 +00003090 prot->twsk_prot->twsk_slab_name = kasprintf(GFP_KERNEL, "tw_sock_%s", prot->name);
Arnaldo Carvalho de Melo8feaf0c02005-08-09 20:09:30 -07003091
Catalin Marinas7e56b5d2008-11-21 16:45:22 -08003092 if (prot->twsk_prot->twsk_slab_name == NULL)
Arnaldo Carvalho de Melo8feaf0c02005-08-09 20:09:30 -07003093 goto out_free_request_sock_slab;
3094
Arnaldo Carvalho de Melo6d6ee432005-12-13 23:25:19 -08003095 prot->twsk_prot->twsk_slab =
Catalin Marinas7e56b5d2008-11-21 16:45:22 -08003096 kmem_cache_create(prot->twsk_prot->twsk_slab_name,
Arnaldo Carvalho de Melo6d6ee432005-12-13 23:25:19 -08003097 prot->twsk_prot->twsk_obj_size,
Eric Dumazet3ab5aee2008-11-16 19:40:17 -08003098 0,
Eric Dumazet52db70d2015-04-10 06:07:18 -07003099 prot->slab_flags,
Paul Mundt20c2df82007-07-20 10:11:58 +09003100 NULL);
Arnaldo Carvalho de Melo6d6ee432005-12-13 23:25:19 -08003101 if (prot->twsk_prot->twsk_slab == NULL)
Arnaldo Carvalho de Melo8feaf0c02005-08-09 20:09:30 -07003102 goto out_free_timewait_sock_slab_name;
3103 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003104 }
3105
Glauber Costa36b77a52011-12-16 00:51:59 +00003106 mutex_lock(&proto_list_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003107 list_add(&prot->node, &proto_list);
Pavel Emelyanov13ff3d62008-03-28 16:38:17 -07003108 assign_proto_idx(prot);
Glauber Costa36b77a52011-12-16 00:51:59 +00003109 mutex_unlock(&proto_list_mutex);
Pavel Emelyanovb733c002007-11-07 02:23:38 -08003110 return 0;
3111
Arnaldo Carvalho de Melo8feaf0c02005-08-09 20:09:30 -07003112out_free_timewait_sock_slab_name:
Catalin Marinas7e56b5d2008-11-21 16:45:22 -08003113 kfree(prot->twsk_prot->twsk_slab_name);
Arnaldo Carvalho de Melo8feaf0c02005-08-09 20:09:30 -07003114out_free_request_sock_slab:
Eric Dumazet0159dfd2015-03-12 16:44:07 -07003115 req_prot_cleanup(prot->rsk_prot);
3116
Arnaldo Carvalho de Melo2e6599c2005-06-18 22:46:52 -07003117 kmem_cache_destroy(prot->slab);
3118 prot->slab = NULL;
Pavel Emelyanovb733c002007-11-07 02:23:38 -08003119out:
3120 return -ENOBUFS;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003121}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003122EXPORT_SYMBOL(proto_register);
3123
3124void proto_unregister(struct proto *prot)
3125{
Glauber Costa36b77a52011-12-16 00:51:59 +00003126 mutex_lock(&proto_list_mutex);
Pavel Emelyanov13ff3d62008-03-28 16:38:17 -07003127 release_proto_idx(prot);
Patrick McHardy0a3f4352005-09-06 19:47:50 -07003128 list_del(&prot->node);
Glauber Costa36b77a52011-12-16 00:51:59 +00003129 mutex_unlock(&proto_list_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003130
Julia Lawalladf78ed2015-09-13 14:15:18 +02003131 kmem_cache_destroy(prot->slab);
3132 prot->slab = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003133
Eric Dumazet0159dfd2015-03-12 16:44:07 -07003134 req_prot_cleanup(prot->rsk_prot);
Arnaldo Carvalho de Melo2e6599c2005-06-18 22:46:52 -07003135
Arnaldo Carvalho de Melo6d6ee432005-12-13 23:25:19 -08003136 if (prot->twsk_prot != NULL && prot->twsk_prot->twsk_slab != NULL) {
Arnaldo Carvalho de Melo6d6ee432005-12-13 23:25:19 -08003137 kmem_cache_destroy(prot->twsk_prot->twsk_slab);
Catalin Marinas7e56b5d2008-11-21 16:45:22 -08003138 kfree(prot->twsk_prot->twsk_slab_name);
Arnaldo Carvalho de Melo6d6ee432005-12-13 23:25:19 -08003139 prot->twsk_prot->twsk_slab = NULL;
Arnaldo Carvalho de Melo8feaf0c02005-08-09 20:09:30 -07003140 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003141}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003142EXPORT_SYMBOL(proto_unregister);
3143
3144#ifdef CONFIG_PROC_FS
Linus Torvalds1da177e2005-04-16 15:20:36 -07003145static void *proto_seq_start(struct seq_file *seq, loff_t *pos)
Glauber Costa36b77a52011-12-16 00:51:59 +00003146 __acquires(proto_list_mutex)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003147{
Glauber Costa36b77a52011-12-16 00:51:59 +00003148 mutex_lock(&proto_list_mutex);
Pavel Emelianov60f04382007-07-09 13:15:14 -07003149 return seq_list_start_head(&proto_list, *pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003150}
3151
3152static void *proto_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3153{
Pavel Emelianov60f04382007-07-09 13:15:14 -07003154 return seq_list_next(v, &proto_list, pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003155}
3156
3157static void proto_seq_stop(struct seq_file *seq, void *v)
Glauber Costa36b77a52011-12-16 00:51:59 +00003158 __releases(proto_list_mutex)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003159{
Glauber Costa36b77a52011-12-16 00:51:59 +00003160 mutex_unlock(&proto_list_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003161}
3162
3163static char proto_method_implemented(const void *method)
3164{
3165 return method == NULL ? 'n' : 'y';
3166}
Glauber Costa180d8cd2011-12-11 21:47:02 +00003167static long sock_prot_memory_allocated(struct proto *proto)
3168{
Jeffrin Josecb75a362012-04-25 19:17:29 +05303169 return proto->memory_allocated != NULL ? proto_memory_allocated(proto) : -1L;
Glauber Costa180d8cd2011-12-11 21:47:02 +00003170}
3171
3172static char *sock_prot_memory_pressure(struct proto *proto)
3173{
3174 return proto->memory_pressure != NULL ?
3175 proto_memory_pressure(proto) ? "yes" : "no" : "NI";
3176}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003177
3178static void proto_seq_printf(struct seq_file *seq, struct proto *proto)
3179{
Glauber Costa180d8cd2011-12-11 21:47:02 +00003180
Eric Dumazet8d987e52010-11-09 23:24:26 +00003181 seq_printf(seq, "%-9s %4u %6d %6ld %-3s %6u %-3s %-10s "
Linus Torvalds1da177e2005-04-16 15:20:36 -07003182 "%2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c\n",
3183 proto->name,
3184 proto->obj_size,
Eric Dumazet14e943d2008-11-19 15:14:01 -08003185 sock_prot_inuse_get(seq_file_net(seq), proto),
Glauber Costa180d8cd2011-12-11 21:47:02 +00003186 sock_prot_memory_allocated(proto),
3187 sock_prot_memory_pressure(proto),
Linus Torvalds1da177e2005-04-16 15:20:36 -07003188 proto->max_header,
3189 proto->slab == NULL ? "no" : "yes",
3190 module_name(proto->owner),
3191 proto_method_implemented(proto->close),
3192 proto_method_implemented(proto->connect),
3193 proto_method_implemented(proto->disconnect),
3194 proto_method_implemented(proto->accept),
3195 proto_method_implemented(proto->ioctl),
3196 proto_method_implemented(proto->init),
3197 proto_method_implemented(proto->destroy),
3198 proto_method_implemented(proto->shutdown),
3199 proto_method_implemented(proto->setsockopt),
3200 proto_method_implemented(proto->getsockopt),
3201 proto_method_implemented(proto->sendmsg),
3202 proto_method_implemented(proto->recvmsg),
3203 proto_method_implemented(proto->sendpage),
3204 proto_method_implemented(proto->bind),
3205 proto_method_implemented(proto->backlog_rcv),
3206 proto_method_implemented(proto->hash),
3207 proto_method_implemented(proto->unhash),
3208 proto_method_implemented(proto->get_port),
3209 proto_method_implemented(proto->enter_memory_pressure));
3210}
3211
3212static int proto_seq_show(struct seq_file *seq, void *v)
3213{
Pavel Emelianov60f04382007-07-09 13:15:14 -07003214 if (v == &proto_list)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003215 seq_printf(seq, "%-9s %-4s %-8s %-6s %-5s %-7s %-4s %-10s %s",
3216 "protocol",
3217 "size",
3218 "sockets",
3219 "memory",
3220 "press",
3221 "maxhdr",
3222 "slab",
3223 "module",
3224 "cl co di ac io in de sh ss gs se re sp bi br ha uh gp em\n");
3225 else
Pavel Emelianov60f04382007-07-09 13:15:14 -07003226 proto_seq_printf(seq, list_entry(v, struct proto, node));
Linus Torvalds1da177e2005-04-16 15:20:36 -07003227 return 0;
3228}
3229
Stephen Hemmingerf6908082007-03-12 14:34:29 -07003230static const struct seq_operations proto_seq_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003231 .start = proto_seq_start,
3232 .next = proto_seq_next,
3233 .stop = proto_seq_stop,
3234 .show = proto_seq_show,
3235};
3236
3237static int proto_seq_open(struct inode *inode, struct file *file)
3238{
Eric Dumazet14e943d2008-11-19 15:14:01 -08003239 return seq_open_net(inode, file, &proto_seq_ops,
3240 sizeof(struct seq_net_private));
Linus Torvalds1da177e2005-04-16 15:20:36 -07003241}
3242
Arjan van de Ven9a321442007-02-12 00:55:35 -08003243static const struct file_operations proto_seq_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003244 .owner = THIS_MODULE,
3245 .open = proto_seq_open,
3246 .read = seq_read,
3247 .llseek = seq_lseek,
Eric Dumazet14e943d2008-11-19 15:14:01 -08003248 .release = seq_release_net,
3249};
3250
3251static __net_init int proto_init_net(struct net *net)
3252{
Gao fengd4beaa62013-02-18 01:34:54 +00003253 if (!proc_create("protocols", S_IRUGO, net->proc_net, &proto_seq_fops))
Eric Dumazet14e943d2008-11-19 15:14:01 -08003254 return -ENOMEM;
3255
3256 return 0;
3257}
3258
3259static __net_exit void proto_exit_net(struct net *net)
3260{
Gao fengece31ff2013-02-18 01:34:56 +00003261 remove_proc_entry("protocols", net->proc_net);
Eric Dumazet14e943d2008-11-19 15:14:01 -08003262}
3263
3264
3265static __net_initdata struct pernet_operations proto_net_ops = {
3266 .init = proto_init_net,
3267 .exit = proto_exit_net,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003268};
3269
3270static int __init proto_init(void)
3271{
Eric Dumazet14e943d2008-11-19 15:14:01 -08003272 return register_pernet_subsys(&proto_net_ops);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003273}
3274
3275subsys_initcall(proto_init);
3276
3277#endif /* PROC_FS */
Sridhar Samudrala7db6b042017-03-24 10:08:24 -07003278
3279#ifdef CONFIG_NET_RX_BUSY_POLL
3280bool sk_busy_loop_end(void *p, unsigned long start_time)
3281{
3282 struct sock *sk = p;
3283
3284 return !skb_queue_empty(&sk->sk_receive_queue) ||
3285 sk_busy_loop_timeout(sk, start_time);
3286}
3287EXPORT_SYMBOL(sk_busy_loop_end);
3288#endif /* CONFIG_NET_RX_BUSY_POLL */