blob: 869637a7caf73abe6b52ef8b0515ca19f5e723c1 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * Definitions for the TCP module.
7 *
8 * Version: @(#)tcp.h 1.0.5 05/23/93
9 *
Jesper Juhl02c30a82005-05-05 16:16:16 -070010 * Authors: Ross Biro
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 *
13 * This program is free software; you can redistribute it and/or
14 * modify it under the terms of the GNU General Public License
15 * as published by the Free Software Foundation; either version
16 * 2 of the License, or (at your option) any later version.
17 */
18#ifndef _TCP_H
19#define _TCP_H
20
Linus Torvalds1da177e2005-04-16 15:20:36 -070021#define FASTRETRANS_DEBUG 1
22
Linus Torvalds1da177e2005-04-16 15:20:36 -070023#include <linux/list.h>
24#include <linux/tcp.h>
Paul Gortmaker187f1882011-11-23 20:12:59 -050025#include <linux/bug.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070026#include <linux/slab.h>
27#include <linux/cache.h>
28#include <linux/percpu.h>
Herbert Xufb286bb2005-11-10 13:01:24 -080029#include <linux/skbuff.h>
YOSHIFUJI Hideakicfb6eeb2006-11-14 19:07:45 -080030#include <linux/crypto.h>
Glenn Griffinc6aefaf2008-02-07 21:49:26 -080031#include <linux/cryptohash.h>
William Allen Simpson435cf552009-12-02 18:17:05 +000032#include <linux/kref.h>
Eric Dumazet740b0f12014-02-26 14:02:48 -080033#include <linux/ktime.h>
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -070034
35#include <net/inet_connection_sock.h>
Arnaldo Carvalho de Melo295ff7e2005-08-09 20:44:40 -070036#include <net/inet_timewait_sock.h>
Arnaldo Carvalho de Melo77d8bf92005-08-09 20:00:51 -070037#include <net/inet_hashtables.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070038#include <net/checksum.h>
Arnaldo Carvalho de Melo2e6599c2005-06-18 22:46:52 -070039#include <net/request_sock.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070040#include <net/sock.h>
41#include <net/snmp.h>
42#include <net/ip.h>
Arnaldo Carvalho de Meloc752f072005-08-09 20:08:28 -070043#include <net/tcp_states.h>
Ilpo Järvinenbdf1ee52007-05-27 02:04:16 -070044#include <net/inet_ecn.h>
Satoru SATOH0c266892009-05-04 11:11:01 -070045#include <net/dst.h>
Arnaldo Carvalho de Meloc752f072005-08-09 20:08:28 -070046
Linus Torvalds1da177e2005-04-16 15:20:36 -070047#include <linux/seq_file.h>
Glauber Costa180d8cd2011-12-11 21:47:02 +000048#include <linux/memcontrol.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070049
Arnaldo Carvalho de Melo6e04e022005-08-09 20:07:35 -070050extern struct inet_hashinfo tcp_hashinfo;
Linus Torvalds1da177e2005-04-16 15:20:36 -070051
Eric Dumazetdd24c002008-11-25 21:17:14 -080052extern struct percpu_counter tcp_orphan_count;
Joe Perches5c9f3022013-09-23 11:33:32 -070053void tcp_time_wait(struct sock *sk, int state, int timeo);
Linus Torvalds1da177e2005-04-16 15:20:36 -070054
Linus Torvalds1da177e2005-04-16 15:20:36 -070055#define MAX_TCP_HEADER (128 + MAX_HEADER)
Adam Langley33ad7982008-07-19 00:04:31 -070056#define MAX_TCP_OPTION_SPACE 40
Linus Torvalds1da177e2005-04-16 15:20:36 -070057
58/*
59 * Never offer a window over 32767 without using window scaling. Some
60 * poor stacks do signed 16bit maths!
61 */
62#define MAX_TCP_WINDOW 32767U
63
64/* Minimal accepted MSS. It is (60+60+8) - (20+20). */
65#define TCP_MIN_MSS 88U
66
John Heffner5d424d52006-03-20 17:53:41 -080067/* The least MTU to use for probing */
68#define TCP_BASE_MSS 512
69
Linus Torvalds1da177e2005-04-16 15:20:36 -070070/* After receiving this amount of duplicate ACKs fast retransmit starts. */
71#define TCP_FASTRETRANS_THRESH 3
72
73/* Maximal reordering. */
74#define TCP_MAX_REORDERING 127
75
76/* Maximal number of ACKs sent quickly to accelerate slow-start. */
77#define TCP_MAX_QUICKACKS 16U
78
79/* urg_data states */
80#define TCP_URG_VALID 0x0100
81#define TCP_URG_NOTYET 0x0200
82#define TCP_URG_READ 0x0400
83
84#define TCP_RETR1 3 /*
85 * This is how many retries it does before it
86 * tries to figure out if the gateway is
87 * down. Minimal RFC value is 3; it corresponds
88 * to ~3sec-8min depending on RTO.
89 */
90
91#define TCP_RETR2 15 /*
92 * This should take at least
93 * 90 minutes to time out.
94 * RFC1122 says that the limit is 100 sec.
95 * 15 is ~13-30min depending on RTO.
96 */
97
Alex Bergmann6c9ff972012-08-31 02:48:31 +000098#define TCP_SYN_RETRIES 6 /* This is how many retries are done
99 * when active opening a connection.
100 * RFC1122 says the minimum retry MUST
101 * be at least 180secs. Nevertheless
102 * this value is corresponding to
103 * 63secs of retransmission with the
104 * current initial RTO.
105 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106
Alex Bergmann6c9ff972012-08-31 02:48:31 +0000107#define TCP_SYNACK_RETRIES 5 /* This is how may retries are done
108 * when passive opening a connection.
109 * This is corresponding to 31secs of
110 * retransmission with the current
111 * initial RTO.
112 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113
Linus Torvalds1da177e2005-04-16 15:20:36 -0700114#define TCP_TIMEWAIT_LEN (60*HZ) /* how long to wait to destroy TIME-WAIT
115 * state, about 60 seconds */
116#define TCP_FIN_TIMEOUT TCP_TIMEWAIT_LEN
117 /* BSD style FIN_WAIT2 deadlock breaker.
118 * It used to be 3min, new value is 60sec,
119 * to combine FIN-WAIT-2 timeout with
120 * TIME-WAIT timer.
121 */
122
123#define TCP_DELACK_MAX ((unsigned)(HZ/5)) /* maximal time to delay before sending an ACK */
124#if HZ >= 100
125#define TCP_DELACK_MIN ((unsigned)(HZ/25)) /* minimal time to delay before sending an ACK */
126#define TCP_ATO_MIN ((unsigned)(HZ/25))
127#else
128#define TCP_DELACK_MIN 4U
129#define TCP_ATO_MIN 4U
130#endif
131#define TCP_RTO_MAX ((unsigned)(120*HZ))
132#define TCP_RTO_MIN ((unsigned)(HZ/5))
Eric Dumazetfd4f2ce2012-04-12 19:48:40 +0000133#define TCP_TIMEOUT_INIT ((unsigned)(1*HZ)) /* RFC6298 2.1 initial RTO value */
Jerry Chu9ad7c042011-06-08 11:08:38 +0000134#define TCP_TIMEOUT_FALLBACK ((unsigned)(3*HZ)) /* RFC 1122 initial RTO value, now
135 * used as a fallback RTO for the
136 * initial data transmission if no
137 * valid RTT sample has been acquired,
138 * most likely due to retrans in 3WHS.
139 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700140
141#define TCP_RESOURCE_PROBE_INTERVAL ((unsigned)(HZ/2U)) /* Maximal interval between probes
142 * for local resources.
143 */
144
145#define TCP_KEEPALIVE_TIME (120*60*HZ) /* two hours */
146#define TCP_KEEPALIVE_PROBES 9 /* Max of 9 keepalive probes */
147#define TCP_KEEPALIVE_INTVL (75*HZ)
148
149#define MAX_TCP_KEEPIDLE 32767
150#define MAX_TCP_KEEPINTVL 32767
151#define MAX_TCP_KEEPCNT 127
152#define MAX_TCP_SYNCNT 127
153
154#define TCP_SYNQ_INTERVAL (HZ/5) /* Period of SYNACK timer */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700155
156#define TCP_PAWS_24DAYS (60 * 60 * 24 * 24)
157#define TCP_PAWS_MSL 60 /* Per-host timestamps are invalidated
158 * after this time. It should be equal
159 * (or greater than) TCP_TIMEWAIT_LEN
160 * to provide reliability equal to one
161 * provided by timewait state.
162 */
163#define TCP_PAWS_WINDOW 1 /* Replay window for per-host
164 * timestamps. It must be less than
165 * minimal timewait lifetime.
166 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700167/*
168 * TCP option
169 */
170
171#define TCPOPT_NOP 1 /* Padding */
172#define TCPOPT_EOL 0 /* End of options */
173#define TCPOPT_MSS 2 /* Segment size negotiating */
174#define TCPOPT_WINDOW 3 /* Window scaling */
175#define TCPOPT_SACK_PERM 4 /* SACK Permitted */
176#define TCPOPT_SACK 5 /* SACK Block */
177#define TCPOPT_TIMESTAMP 8 /* Better RTT estimations/PAWS */
YOSHIFUJI Hideakicfb6eeb2006-11-14 19:07:45 -0800178#define TCPOPT_MD5SIG 19 /* MD5 Signature (RFC2385) */
Yuchung Cheng2100c8d2012-07-19 06:43:05 +0000179#define TCPOPT_EXP 254 /* Experimental */
180/* Magic number to be after the option value for sharing TCP
181 * experimental options. See draft-ietf-tcpm-experimental-options-00.txt
182 */
183#define TCPOPT_FASTOPEN_MAGIC 0xF989
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184
185/*
186 * TCP option lengths
187 */
188
189#define TCPOLEN_MSS 4
190#define TCPOLEN_WINDOW 3
191#define TCPOLEN_SACK_PERM 2
192#define TCPOLEN_TIMESTAMP 10
YOSHIFUJI Hideakicfb6eeb2006-11-14 19:07:45 -0800193#define TCPOLEN_MD5SIG 18
Yuchung Cheng2100c8d2012-07-19 06:43:05 +0000194#define TCPOLEN_EXP_FASTOPEN_BASE 4
Linus Torvalds1da177e2005-04-16 15:20:36 -0700195
196/* But this is what stacks really send out. */
197#define TCPOLEN_TSTAMP_ALIGNED 12
198#define TCPOLEN_WSCALE_ALIGNED 4
199#define TCPOLEN_SACKPERM_ALIGNED 4
200#define TCPOLEN_SACK_BASE 2
201#define TCPOLEN_SACK_BASE_ALIGNED 4
202#define TCPOLEN_SACK_PERBLOCK 8
YOSHIFUJI Hideakicfb6eeb2006-11-14 19:07:45 -0800203#define TCPOLEN_MD5SIG_ALIGNED 20
Adam Langley33ad7982008-07-19 00:04:31 -0700204#define TCPOLEN_MSS_ALIGNED 4
Linus Torvalds1da177e2005-04-16 15:20:36 -0700205
Linus Torvalds1da177e2005-04-16 15:20:36 -0700206/* Flags in tp->nonagle */
207#define TCP_NAGLE_OFF 1 /* Nagle's algo is disabled */
208#define TCP_NAGLE_CORK 2 /* Socket is corked */
Stephen Hemmingercaa20d9a2005-11-10 17:13:47 -0800209#define TCP_NAGLE_PUSH 4 /* Cork is overridden for already queued data */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700210
Andreas Petlund36e31b0a2010-02-18 02:47:01 +0000211/* TCP thin-stream limits */
212#define TCP_THIN_LINEAR_RETRIES 6 /* After 6 linear retries, do exp. backoff */
213
David S. Miller7eb38522011-02-05 18:13:45 -0800214/* TCP initial congestion window as per draft-hkchu-tcpm-initcwnd-01 */
David S. Miller442b9632011-02-02 17:05:11 -0800215#define TCP_INIT_CWND 10
216
Yuchung Chengcf60af02012-07-19 06:43:09 +0000217/* Bit Flags for sysctl_tcp_fastopen */
218#define TFO_CLIENT_ENABLE 1
Jerry Chu10467162012-08-31 12:29:11 +0000219#define TFO_SERVER_ENABLE 2
Yuchung Cheng67da22d2012-07-19 06:43:11 +0000220#define TFO_CLIENT_NO_COOKIE 4 /* Data in SYN w/o cookie option */
Yuchung Chengcf60af02012-07-19 06:43:09 +0000221
Jerry Chu10467162012-08-31 12:29:11 +0000222/* Accept SYN data w/o any cookie option */
223#define TFO_SERVER_COOKIE_NOT_REQD 0x200
224
225/* Force enable TFO on all listeners, i.e., not requiring the
226 * TCP_FASTOPEN socket option. SOCKOPT1/2 determine how to set max_qlen.
227 */
228#define TFO_SERVER_WO_SOCKOPT1 0x400
229#define TFO_SERVER_WO_SOCKOPT2 0x800
Jerry Chu10467162012-08-31 12:29:11 +0000230
Arnaldo Carvalho de Melo295ff7e2005-08-09 20:44:40 -0700231extern struct inet_timewait_death_row tcp_death_row;
232
Linus Torvalds1da177e2005-04-16 15:20:36 -0700233/* sysctl variables for tcp */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700234extern int sysctl_tcp_timestamps;
235extern int sysctl_tcp_window_scaling;
236extern int sysctl_tcp_sack;
237extern int sysctl_tcp_fin_timeout;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700238extern int sysctl_tcp_keepalive_time;
239extern int sysctl_tcp_keepalive_probes;
240extern int sysctl_tcp_keepalive_intvl;
241extern int sysctl_tcp_syn_retries;
242extern int sysctl_tcp_synack_retries;
243extern int sysctl_tcp_retries1;
244extern int sysctl_tcp_retries2;
245extern int sysctl_tcp_orphan_retries;
246extern int sysctl_tcp_syncookies;
Yuchung Cheng2100c8d2012-07-19 06:43:05 +0000247extern int sysctl_tcp_fastopen;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700248extern int sysctl_tcp_retrans_collapse;
249extern int sysctl_tcp_stdurg;
250extern int sysctl_tcp_rfc1337;
251extern int sysctl_tcp_abort_on_overflow;
252extern int sysctl_tcp_max_orphans;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700253extern int sysctl_tcp_fack;
254extern int sysctl_tcp_reordering;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700255extern int sysctl_tcp_dsack;
Eric W. Biedermana4fe34b2013-10-19 16:25:36 -0700256extern long sysctl_tcp_mem[3];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700257extern int sysctl_tcp_wmem[3];
258extern int sysctl_tcp_rmem[3];
259extern int sysctl_tcp_app_win;
260extern int sysctl_tcp_adv_win_scale;
261extern int sysctl_tcp_tw_reuse;
262extern int sysctl_tcp_frto;
263extern int sysctl_tcp_low_latency;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700264extern int sysctl_tcp_nometrics_save;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700265extern int sysctl_tcp_moderate_rcvbuf;
266extern int sysctl_tcp_tso_win_divisor;
John Heffner5d424d52006-03-20 17:53:41 -0800267extern int sysctl_tcp_mtu_probing;
268extern int sysctl_tcp_base_mss;
Rick Jones15d99e02006-03-20 22:40:29 -0800269extern int sysctl_tcp_workaround_signed_windows;
David S. Miller35089bb2006-06-13 22:33:04 -0700270extern int sysctl_tcp_slow_start_after_idle;
Andreas Petlund36e31b0a2010-02-18 02:47:01 +0000271extern int sysctl_tcp_thin_linear_timeouts;
Andreas Petlund7e380172010-02-18 04:48:19 +0000272extern int sysctl_tcp_thin_dupack;
Yuchung Chengeed530b2012-05-02 13:30:03 +0000273extern int sysctl_tcp_early_retrans;
Eric Dumazet46d3cea2012-07-11 05:50:31 +0000274extern int sysctl_tcp_limit_output_bytes;
Eric Dumazet282f23c2012-07-17 10:13:05 +0200275extern int sysctl_tcp_challenge_ack_limit;
Eric Dumazetc9bee3b72013-07-22 20:27:07 -0700276extern unsigned int sysctl_tcp_notsent_lowat;
Eric Dumazet95bd09e2013-08-27 05:46:32 -0700277extern int sysctl_tcp_min_tso_segs;
Eric Dumazetf54b3112013-12-05 22:36:05 -0800278extern int sysctl_tcp_autocorking;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700279
Eric Dumazet8d987e52010-11-09 23:24:26 +0000280extern atomic_long_t tcp_memory_allocated;
Eric Dumazet17483762008-11-25 21:16:35 -0800281extern struct percpu_counter tcp_sockets_allocated;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700282extern int tcp_memory_pressure;
283
Linus Torvalds1da177e2005-04-16 15:20:36 -0700284/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700285 * The next routines deal with comparing 32 bit unsigned ints
286 * and worry about wraparound (automatic with unsigned arithmetic).
287 */
288
Eric Dumazeta2a385d2012-05-16 23:15:34 +0000289static inline bool before(__u32 seq1, __u32 seq2)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700290{
Gerrit Renker0d630cc2007-01-04 12:25:16 -0800291 return (__s32)(seq1-seq2) < 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700292}
Gerrit Renker9a036b9c2006-12-20 10:25:55 -0800293#define after(seq2, seq1) before(seq1, seq2)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700294
295/* is s2<=s1<=s3 ? */
Eric Dumazeta2a385d2012-05-16 23:15:34 +0000296static inline bool between(__u32 seq1, __u32 seq2, __u32 seq3)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700297{
298 return seq3 - seq2 >= seq1 - seq2;
299}
300
Arun Sharmaefcdbf22012-01-30 14:16:06 -0800301static inline bool tcp_out_of_memory(struct sock *sk)
302{
303 if (sk->sk_wmem_queued > SOCK_MIN_SNDBUF &&
304 sk_memory_allocated(sk) > sk_prot_mem_limits(sk, 2))
305 return true;
306 return false;
307}
308
David S. Millerad1af0f2010-08-25 02:27:49 -0700309static inline bool tcp_too_many_orphans(struct sock *sk, int shift)
Pavel Emelianove4fd5da2007-05-29 13:19:18 -0700310{
David S. Millerad1af0f2010-08-25 02:27:49 -0700311 struct percpu_counter *ocp = sk->sk_prot->orphan_count;
312 int orphans = percpu_counter_read_positive(ocp);
313
314 if (orphans << shift > sysctl_tcp_max_orphans) {
315 orphans = percpu_counter_sum_positive(ocp);
316 if (orphans << shift > sysctl_tcp_max_orphans)
317 return true;
318 }
David S. Millerad1af0f2010-08-25 02:27:49 -0700319 return false;
Pavel Emelianove4fd5da2007-05-29 13:19:18 -0700320}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700321
Joe Perches5c9f3022013-09-23 11:33:32 -0700322bool tcp_check_oom(struct sock *sk, int shift);
Arun Sharmaefcdbf22012-01-30 14:16:06 -0800323
Florian Westphala0f82f62009-04-19 09:43:48 +0000324/* syncookies: remember time of last synqueue overflow */
325static inline void tcp_synq_overflow(struct sock *sk)
326{
327 tcp_sk(sk)->rx_opt.ts_recent_stamp = jiffies;
328}
329
330/* syncookies: no recent synqueue overflow on this listening socket? */
Eric Dumazeta2a385d2012-05-16 23:15:34 +0000331static inline bool tcp_synq_no_recent_overflow(const struct sock *sk)
Florian Westphala0f82f62009-04-19 09:43:48 +0000332{
333 unsigned long last_overflow = tcp_sk(sk)->rx_opt.ts_recent_stamp;
Jerry Chu9ad7c042011-06-08 11:08:38 +0000334 return time_after(jiffies, last_overflow + TCP_TIMEOUT_FALLBACK);
Florian Westphala0f82f62009-04-19 09:43:48 +0000335}
336
Linus Torvalds1da177e2005-04-16 15:20:36 -0700337extern struct proto tcp_prot;
338
Pavel Emelyanov57ef42d2008-07-18 04:02:08 -0700339#define TCP_INC_STATS(net, field) SNMP_INC_STATS((net)->mib.tcp_statistics, field)
340#define TCP_INC_STATS_BH(net, field) SNMP_INC_STATS_BH((net)->mib.tcp_statistics, field)
341#define TCP_DEC_STATS(net, field) SNMP_DEC_STATS((net)->mib.tcp_statistics, field)
342#define TCP_ADD_STATS_USER(net, field, val) SNMP_ADD_STATS_USER((net)->mib.tcp_statistics, field, val)
Tom Herbertaa2ea052010-04-22 07:00:24 +0000343#define TCP_ADD_STATS(net, field, val) SNMP_ADD_STATS((net)->mib.tcp_statistics, field, val)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700344
Joe Perches5c9f3022013-09-23 11:33:32 -0700345void tcp_tasklet_init(void);
Eric Dumazet46d3cea2012-07-11 05:50:31 +0000346
Joe Perches5c9f3022013-09-23 11:33:32 -0700347void tcp_v4_err(struct sk_buff *skb, u32);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700348
Joe Perches5c9f3022013-09-23 11:33:32 -0700349void tcp_shutdown(struct sock *sk, int how);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700350
Joe Perches5c9f3022013-09-23 11:33:32 -0700351void tcp_v4_early_demux(struct sk_buff *skb);
352int tcp_v4_rcv(struct sk_buff *skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700353
Joe Perches5c9f3022013-09-23 11:33:32 -0700354int tcp_v4_tw_remember_stamp(struct inet_timewait_sock *tw);
355int tcp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
356 size_t size);
357int tcp_sendpage(struct sock *sk, struct page *page, int offset, size_t size,
358 int flags);
359void tcp_release_cb(struct sock *sk);
360void tcp_wfree(struct sk_buff *skb);
361void tcp_write_timer_handler(struct sock *sk);
362void tcp_delack_timer_handler(struct sock *sk);
363int tcp_ioctl(struct sock *sk, int cmd, unsigned long arg);
364int tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb,
365 const struct tcphdr *th, unsigned int len);
366void tcp_rcv_established(struct sock *sk, struct sk_buff *skb,
367 const struct tcphdr *th, unsigned int len);
368void tcp_rcv_space_adjust(struct sock *sk);
Joe Perches5c9f3022013-09-23 11:33:32 -0700369int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp);
370void tcp_twsk_destructor(struct sock *sk);
371ssize_t tcp_splice_read(struct socket *sk, loff_t *ppos,
372 struct pipe_inode_info *pipe, size_t len,
373 unsigned int flags);
Jens Axboe9c55e012007-11-06 23:30:13 -0800374
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -0700375static inline void tcp_dec_quickack_mode(struct sock *sk,
376 const unsigned int pkts)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700377{
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -0700378 struct inet_connection_sock *icsk = inet_csk(sk);
David S. Millerfc6415bc2005-07-05 15:17:45 -0700379
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -0700380 if (icsk->icsk_ack.quick) {
381 if (pkts >= icsk->icsk_ack.quick) {
382 icsk->icsk_ack.quick = 0;
David S. Millerfc6415bc2005-07-05 15:17:45 -0700383 /* Leaving quickack mode we deflate ATO. */
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -0700384 icsk->icsk_ack.ato = TCP_ATO_MIN;
David S. Millerfc6415bc2005-07-05 15:17:45 -0700385 } else
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -0700386 icsk->icsk_ack.quick -= pkts;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700387 }
388}
389
Ilpo Järvinenbdf1ee52007-05-27 02:04:16 -0700390#define TCP_ECN_OK 1
391#define TCP_ECN_QUEUE_CWR 2
392#define TCP_ECN_DEMAND_CWR 4
Eric Dumazet7a269ff2011-09-22 20:02:19 +0000393#define TCP_ECN_SEEN 8
Ilpo Järvinenbdf1ee52007-05-27 02:04:16 -0700394
Eric Dumazetfd2c3ef2009-11-03 03:26:03 +0000395enum tcp_tw_status {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700396 TCP_TW_SUCCESS = 0,
397 TCP_TW_RST = 1,
398 TCP_TW_ACK = 2,
399 TCP_TW_SYN = 3
400};
401
402
Joe Perches5c9f3022013-09-23 11:33:32 -0700403enum tcp_tw_status tcp_timewait_state_process(struct inet_timewait_sock *tw,
404 struct sk_buff *skb,
405 const struct tcphdr *th);
406struct sock *tcp_check_req(struct sock *sk, struct sk_buff *skb,
407 struct request_sock *req, struct request_sock **prev,
408 bool fastopen);
409int tcp_child_process(struct sock *parent, struct sock *child,
410 struct sk_buff *skb);
Neal Cardwell5ae344c2014-08-04 19:12:29 -0400411void tcp_enter_loss(struct sock *sk);
Joe Perches5c9f3022013-09-23 11:33:32 -0700412void tcp_clear_retrans(struct tcp_sock *tp);
413void tcp_update_metrics(struct sock *sk);
414void tcp_init_metrics(struct sock *sk);
415void tcp_metrics_init(void);
416bool tcp_peer_is_proven(struct request_sock *req, struct dst_entry *dst,
Hannes Frederic Sowaa26552a2014-08-14 22:06:12 +0200417 bool paws_check, bool timestamps);
Joe Perches5c9f3022013-09-23 11:33:32 -0700418bool tcp_remember_stamp(struct sock *sk);
419bool tcp_tw_remember_stamp(struct inet_timewait_sock *tw);
420void tcp_fetch_timewait_stamp(struct sock *sk, struct dst_entry *dst);
421void tcp_disable_fack(struct tcp_sock *tp);
422void tcp_close(struct sock *sk, long timeout);
423void tcp_init_sock(struct sock *sk);
424unsigned int tcp_poll(struct file *file, struct socket *sock,
425 struct poll_table_struct *wait);
426int tcp_getsockopt(struct sock *sk, int level, int optname,
427 char __user *optval, int __user *optlen);
428int tcp_setsockopt(struct sock *sk, int level, int optname,
429 char __user *optval, unsigned int optlen);
430int compat_tcp_getsockopt(struct sock *sk, int level, int optname,
Changli Gao53d31762010-07-10 20:41:06 +0000431 char __user *optval, int __user *optlen);
Joe Perches5c9f3022013-09-23 11:33:32 -0700432int compat_tcp_setsockopt(struct sock *sk, int level, int optname,
Changli Gao53d31762010-07-10 20:41:06 +0000433 char __user *optval, unsigned int optlen);
Joe Perches5c9f3022013-09-23 11:33:32 -0700434void tcp_set_keepalive(struct sock *sk, int val);
435void tcp_syn_ack_timeout(struct sock *sk, struct request_sock *req);
436int tcp_recvmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
437 size_t len, int nonblock, int flags, int *addr_len);
438void tcp_parse_options(const struct sk_buff *skb,
439 struct tcp_options_received *opt_rx,
440 int estab, struct tcp_fastopen_cookie *foc);
441const u8 *tcp_parse_md5sig_option(const struct tcphdr *th);
YOSHIFUJI Hideaki7d5d5522008-04-17 12:29:53 +0900442
Linus Torvalds1da177e2005-04-16 15:20:36 -0700443/*
444 * TCP v4 functions exported for the inet6 API
445 */
446
Joe Perches5c9f3022013-09-23 11:33:32 -0700447void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb);
Neal Cardwell4fab9072014-08-14 12:40:05 -0400448void tcp_v4_mtu_reduced(struct sock *sk);
Joe Perches5c9f3022013-09-23 11:33:32 -0700449int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb);
450struct sock *tcp_create_openreq_child(struct sock *sk,
451 struct request_sock *req,
452 struct sk_buff *skb);
453struct sock *tcp_v4_syn_recv_sock(struct sock *sk, struct sk_buff *skb,
454 struct request_sock *req,
455 struct dst_entry *dst);
456int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb);
457int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len);
458int tcp_connect(struct sock *sk);
459struct sk_buff *tcp_make_synack(struct sock *sk, struct dst_entry *dst,
460 struct request_sock *req,
461 struct tcp_fastopen_cookie *foc);
462int tcp_disconnect(struct sock *sk, int flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700463
Pavel Emelyanov370816a2012-04-19 03:40:01 +0000464void tcp_finish_connect(struct sock *sk, struct sk_buff *skb);
Pavel Emelyanov292e8d82012-05-10 01:49:41 +0000465int tcp_send_rcvq(struct sock *sk, struct msghdr *msg, size_t size);
Eric Dumazet63d02d12012-08-09 14:11:00 +0000466void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700467
Linus Torvalds1da177e2005-04-16 15:20:36 -0700468/* From syncookies.c */
Joe Perches5c9f3022013-09-23 11:33:32 -0700469int __cookie_v4_check(const struct iphdr *iph, const struct tcphdr *th,
470 u32 cookie);
471struct sock *cookie_v4_check(struct sock *sk, struct sk_buff *skb,
472 struct ip_options *opt);
Eric Dumazete05c82d2011-09-18 21:02:55 -0400473#ifdef CONFIG_SYN_COOKIES
Florian Westphal8c27bd72013-09-20 22:32:55 +0200474
Eric Dumazet63262312014-03-19 21:02:21 -0700475/* Syncookies use a monotonic timer which increments every 60 seconds.
Florian Westphal8c27bd72013-09-20 22:32:55 +0200476 * This counter is used both as a hash input and partially encoded into
477 * the cookie value. A cookie is only validated further if the delta
478 * between the current counter value and the encoded one is less than this,
Eric Dumazet63262312014-03-19 21:02:21 -0700479 * i.e. a sent cookie is valid only at most for 2*60 seconds (or less if
Florian Westphal8c27bd72013-09-20 22:32:55 +0200480 * the counter advances immediately after a cookie is generated).
481 */
482#define MAX_SYNCOOKIE_AGE 2
483
484static inline u32 tcp_cookie_time(void)
485{
Eric Dumazet63262312014-03-19 21:02:21 -0700486 u64 val = get_jiffies_64();
487
488 do_div(val, 60 * HZ);
489 return val;
Florian Westphal8c27bd72013-09-20 22:32:55 +0200490}
491
Joe Perches5c9f3022013-09-23 11:33:32 -0700492u32 __cookie_v4_init_sequence(const struct iphdr *iph, const struct tcphdr *th,
493 u16 *mssp);
Octavian Purdila57b47552014-06-25 17:09:50 +0300494__u32 cookie_v4_init_sequence(struct sock *sk, const struct sk_buff *skb,
495 __u16 *mss);
Eric Dumazete05c82d2011-09-18 21:02:55 -0400496#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700497
Joe Perches5c9f3022013-09-23 11:33:32 -0700498__u32 cookie_init_timestamp(struct request_sock *req);
499bool cookie_check_timestamp(struct tcp_options_received *opt, struct net *net,
500 bool *ecn_ok);
Florian Westphal4dfc2812008-04-10 03:12:40 -0700501
Glenn Griffinc6aefaf2008-02-07 21:49:26 -0800502/* From net/ipv6/syncookies.c */
Joe Perches5c9f3022013-09-23 11:33:32 -0700503int __cookie_v6_check(const struct ipv6hdr *iph, const struct tcphdr *th,
504 u32 cookie);
505struct sock *cookie_v6_check(struct sock *sk, struct sk_buff *skb);
Eric Dumazete05c82d2011-09-18 21:02:55 -0400506#ifdef CONFIG_SYN_COOKIES
Joe Perches5c9f3022013-09-23 11:33:32 -0700507u32 __cookie_v6_init_sequence(const struct ipv6hdr *iph,
508 const struct tcphdr *th, u16 *mssp);
509__u32 cookie_v6_init_sequence(struct sock *sk, const struct sk_buff *skb,
510 __u16 *mss);
Eric Dumazete05c82d2011-09-18 21:02:55 -0400511#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700512/* tcp_output.c */
513
Joe Perches5c9f3022013-09-23 11:33:32 -0700514void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss,
515 int nonagle);
516bool tcp_may_send_now(struct sock *sk);
517int __tcp_retransmit_skb(struct sock *, struct sk_buff *);
518int tcp_retransmit_skb(struct sock *, struct sk_buff *);
519void tcp_retransmit_timer(struct sock *sk);
520void tcp_xmit_retransmit_queue(struct sock *);
521void tcp_simple_retransmit(struct sock *);
522int tcp_trim_head(struct sock *, struct sk_buff *, u32);
Octavian Purdila6cc55e02014-06-06 17:32:37 +0300523int tcp_fragment(struct sock *, struct sk_buff *, u32, unsigned int, gfp_t);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700524
Joe Perches5c9f3022013-09-23 11:33:32 -0700525void tcp_send_probe0(struct sock *);
526void tcp_send_partial(struct sock *);
527int tcp_write_wakeup(struct sock *);
528void tcp_send_fin(struct sock *sk);
529void tcp_send_active_reset(struct sock *sk, gfp_t priority);
530int tcp_send_synack(struct sock *);
531bool tcp_syn_flood_action(struct sock *sk, const struct sk_buff *skb,
532 const char *proto);
533void tcp_push_one(struct sock *, unsigned int mss_now);
534void tcp_send_ack(struct sock *sk);
535void tcp_send_delayed_ack(struct sock *sk);
536void tcp_send_loss_probe(struct sock *sk);
537bool tcp_schedule_loss_probe(struct sock *sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700538
David S. Millera762a982005-07-05 15:18:51 -0700539/* tcp_input.c */
Joe Perches5c9f3022013-09-23 11:33:32 -0700540void tcp_resume_early_retransmit(struct sock *sk);
541void tcp_rearm_rto(struct sock *sk);
542void tcp_reset(struct sock *sk);
David S. Millera762a982005-07-05 15:18:51 -0700543
Linus Torvalds1da177e2005-04-16 15:20:36 -0700544/* tcp_timer.c */
Joe Perches5c9f3022013-09-23 11:33:32 -0700545void tcp_init_xmit_timers(struct sock *);
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -0700546static inline void tcp_clear_xmit_timers(struct sock *sk)
547{
548 inet_csk_clear_xmit_timers(sk);
549}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700550
Joe Perches5c9f3022013-09-23 11:33:32 -0700551unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu);
552unsigned int tcp_current_mss(struct sock *sk);
Ilpo Järvinen0c54b852009-03-14 14:23:05 +0000553
554/* Bound MSS / TSO packet size with the half of the window */
555static inline int tcp_bound_to_half_wnd(struct tcp_sock *tp, int pktsize)
556{
Alexey Kuznetsov01f83d62010-09-15 10:27:52 -0700557 int cutoff;
558
559 /* When peer uses tiny windows, there is no use in packetizing
560 * to sub-MSS pieces for the sake of SWS or making sure there
561 * are enough packets in the pipe for fast recovery.
562 *
563 * On the other hand, for extremely large MSS devices, handling
564 * smaller than MSS windows in this way does make sense.
565 */
566 if (tp->max_window >= 512)
567 cutoff = (tp->max_window >> 1);
568 else
569 cutoff = tp->max_window;
570
571 if (cutoff && pktsize > cutoff)
572 return max_t(int, cutoff, 68U - tp->tcp_header_len);
Ilpo Järvinen0c54b852009-03-14 14:23:05 +0000573 else
574 return pktsize;
575}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700576
Arnaldo Carvalho de Melo17b085e2005-08-12 12:59:17 -0300577/* tcp.c */
Joe Perches5c9f3022013-09-23 11:33:32 -0700578void tcp_get_info(const struct sock *, struct tcp_info *);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700579
580/* Read 'sendfile()'-style from a TCP socket */
581typedef int (*sk_read_actor_t)(read_descriptor_t *, struct sk_buff *,
582 unsigned int, size_t);
Joe Perches5c9f3022013-09-23 11:33:32 -0700583int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
584 sk_read_actor_t recv_actor);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700585
Joe Perches5c9f3022013-09-23 11:33:32 -0700586void tcp_initialize_rcv_mss(struct sock *sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700587
Joe Perches5c9f3022013-09-23 11:33:32 -0700588int tcp_mtu_to_mss(struct sock *sk, int pmtu);
589int tcp_mss_to_mtu(struct sock *sk, int mss);
590void tcp_mtup_init(struct sock *sk);
591void tcp_init_buffer_space(struct sock *sk);
John Heffner5d424d52006-03-20 17:53:41 -0800592
Damian Lukowskif1ecd5d2009-08-26 00:16:31 +0000593static inline void tcp_bound_rto(const struct sock *sk)
594{
595 if (inet_csk(sk)->icsk_rto > TCP_RTO_MAX)
596 inet_csk(sk)->icsk_rto = TCP_RTO_MAX;
597}
598
599static inline u32 __tcp_set_rto(const struct tcp_sock *tp)
600{
Eric Dumazet740b0f12014-02-26 14:02:48 -0800601 return usecs_to_jiffies((tp->srtt_us >> 3) + tp->rttvar_us);
Damian Lukowskif1ecd5d2009-08-26 00:16:31 +0000602}
603
Stephen Hemminger40efc6f2006-01-03 16:03:49 -0800604static inline void __tcp_fast_path_on(struct tcp_sock *tp, u32 snd_wnd)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700605{
606 tp->pred_flags = htonl((tp->tcp_header_len << 26) |
607 ntohl(TCP_FLAG_ACK) |
608 snd_wnd);
609}
610
Stephen Hemminger40efc6f2006-01-03 16:03:49 -0800611static inline void tcp_fast_path_on(struct tcp_sock *tp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700612{
613 __tcp_fast_path_on(tp, tp->snd_wnd >> tp->rx_opt.snd_wscale);
614}
615
Ilpo Järvinen9e412ba2007-04-20 22:18:02 -0700616static inline void tcp_fast_path_check(struct sock *sk)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700617{
Ilpo Järvinen9e412ba2007-04-20 22:18:02 -0700618 struct tcp_sock *tp = tcp_sk(sk);
619
David S. Millerb03efcf2005-07-08 14:57:23 -0700620 if (skb_queue_empty(&tp->out_of_order_queue) &&
Linus Torvalds1da177e2005-04-16 15:20:36 -0700621 tp->rcv_wnd &&
622 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf &&
623 !tp->urg_data)
624 tcp_fast_path_on(tp);
625}
626
Satoru SATOH0c266892009-05-04 11:11:01 -0700627/* Compute the actual rto_min value */
628static inline u32 tcp_rto_min(struct sock *sk)
629{
Eric Dumazetcf533ea2011-10-21 05:22:42 -0400630 const struct dst_entry *dst = __sk_dst_get(sk);
Satoru SATOH0c266892009-05-04 11:11:01 -0700631 u32 rto_min = TCP_RTO_MIN;
632
633 if (dst && dst_metric_locked(dst, RTAX_RTO_MIN))
634 rto_min = dst_metric_rtt(dst, RTAX_RTO_MIN);
635 return rto_min;
636}
637
Eric Dumazet740b0f12014-02-26 14:02:48 -0800638static inline u32 tcp_rto_min_us(struct sock *sk)
639{
640 return jiffies_to_usecs(tcp_rto_min(sk));
641}
642
Linus Torvalds1da177e2005-04-16 15:20:36 -0700643/* Compute the actual receive window we are currently advertising.
644 * Rcv_nxt can be after the window if our peer push more data
645 * than the offered window.
646 */
Stephen Hemminger40efc6f2006-01-03 16:03:49 -0800647static inline u32 tcp_receive_window(const struct tcp_sock *tp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700648{
649 s32 win = tp->rcv_wup + tp->rcv_wnd - tp->rcv_nxt;
650
651 if (win < 0)
652 win = 0;
653 return (u32) win;
654}
655
656/* Choose a new window, without checks for shrinking, and without
657 * scaling applied to the result. The caller does these things
658 * if necessary. This is a "raw" window selection.
659 */
Joe Perches5c9f3022013-09-23 11:33:32 -0700660u32 __tcp_select_window(struct sock *sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700661
Pavel Emelyanovee995282012-04-19 03:40:39 +0000662void tcp_send_window_probe(struct sock *sk);
663
Linus Torvalds1da177e2005-04-16 15:20:36 -0700664/* TCP timestamps are only 32-bits, this causes a slight
665 * complication on 64-bit systems since we store a snapshot
Stephen Hemminger31f34262005-11-15 15:17:10 -0800666 * of jiffies in the buffer control blocks below. We decided
667 * to use only the low 32-bits of jiffies and hide the ugly
Linus Torvalds1da177e2005-04-16 15:20:36 -0700668 * casts with the following macro.
669 */
670#define tcp_time_stamp ((__u32)(jiffies))
671
Eric Dumazet7faee5c2014-09-05 15:33:33 -0700672static inline u32 tcp_skb_timestamp(const struct sk_buff *skb)
673{
674 return skb->skb_mstamp.stamp_jiffies;
675}
676
677
Changli Gaoa3433f32010-06-12 14:01:43 +0000678#define tcp_flag_byte(th) (((u_int8_t *)th)[13])
679
680#define TCPHDR_FIN 0x01
681#define TCPHDR_SYN 0x02
682#define TCPHDR_RST 0x04
683#define TCPHDR_PSH 0x08
684#define TCPHDR_ACK 0x10
685#define TCPHDR_URG 0x20
686#define TCPHDR_ECE 0x40
687#define TCPHDR_CWR 0x80
688
Stephen Hemmingercaa20d9a2005-11-10 17:13:47 -0800689/* This is what the send packet queuing engine uses to pass
Eric Dumazetf86586f2010-07-15 21:41:00 -0700690 * TCP per-packet control information to the transmission code.
691 * We also store the host-order sequence numbers in here too.
692 * This is 44 bytes if IPV6 is enabled.
693 * If this grows please adjust skbuff.h:skbuff->cb[xxx] size appropriately.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700694 */
695struct tcp_skb_cb {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700696 __u32 seq; /* Starting sequence number */
697 __u32 end_seq; /* SEQ + FIN + SYN + datalen */
Eric Dumazetcd7d8492014-09-24 04:11:22 -0700698 union {
699 /* Note : tcp_tw_isn is used in input path only
700 * (isn chosen by tcp_timewait_state_process())
701 *
702 * tcp_gso_segs is used in write queue only,
703 * cf tcp_skb_pcount()
704 */
705 __u32 tcp_tw_isn;
706 __u32 tcp_gso_segs;
707 };
Eric Dumazet4de075e2011-09-27 13:25:05 -0400708 __u8 tcp_flags; /* TCP header flags. (tcp[13]) */
Neal Cardwellf4f9f6e2012-04-16 07:08:06 +0000709
Linus Torvalds1da177e2005-04-16 15:20:36 -0700710 __u8 sacked; /* State flags for SACK/FACK. */
711#define TCPCB_SACKED_ACKED 0x01 /* SKB ACK'd by a SACK block */
712#define TCPCB_SACKED_RETRANS 0x02 /* SKB retransmitted */
713#define TCPCB_LOST 0x04 /* SKB is lost */
714#define TCPCB_TAGBITS 0x07 /* All tag bits */
Andrey Vagin9d186ca2014-08-13 16:03:10 +0400715#define TCPCB_REPAIRED 0x10 /* SKB repaired (no skb_mstamp) */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700716#define TCPCB_EVER_RETRANS 0x80 /* Ever retransmitted frame */
Andrey Vagin9d186ca2014-08-13 16:03:10 +0400717#define TCPCB_RETRANS (TCPCB_SACKED_RETRANS|TCPCB_EVER_RETRANS| \
718 TCPCB_REPAIRED)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700719
Neal Cardwellf4f9f6e2012-04-16 07:08:06 +0000720 __u8 ip_dsfield; /* IPv4 tos or IPv6 dsfield */
721 /* 1 byte hole */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700722 __u32 ack_seq; /* Sequence number ACK'd */
Eric Dumazet971f10e2014-09-27 09:50:57 -0700723 union {
724 struct inet_skb_parm h4;
725#if IS_ENABLED(CONFIG_IPV6)
726 struct inet6_skb_parm h6;
727#endif
728 } header; /* For incoming frames */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700729};
730
731#define TCP_SKB_CB(__skb) ((struct tcp_skb_cb *)&((__skb)->cb[0]))
732
Linus Torvalds1da177e2005-04-16 15:20:36 -0700733/* Due to TSO, an SKB can be composed of multiple actual
734 * packets. To keep these tracked properly, we use this.
735 */
736static inline int tcp_skb_pcount(const struct sk_buff *skb)
737{
Eric Dumazetcd7d8492014-09-24 04:11:22 -0700738 return TCP_SKB_CB(skb)->tcp_gso_segs;
739}
740
741static inline void tcp_skb_pcount_set(struct sk_buff *skb, int segs)
742{
743 TCP_SKB_CB(skb)->tcp_gso_segs = segs;
744}
745
746static inline void tcp_skb_pcount_add(struct sk_buff *skb, int segs)
747{
748 TCP_SKB_CB(skb)->tcp_gso_segs += segs;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700749}
750
751/* This is valid iff tcp_skb_pcount() > 1. */
752static inline int tcp_skb_mss(const struct sk_buff *skb)
753{
Herbert Xu79671682006-06-22 02:40:14 -0700754 return skb_shinfo(skb)->gso_size;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700755}
756
Stephen Hemminger317a76f2005-06-23 12:19:55 -0700757/* Events passed to congestion control interface */
758enum tcp_ca_event {
759 CA_EVENT_TX_START, /* first transmit when no packets in flight */
760 CA_EVENT_CWND_RESTART, /* congestion window restart */
761 CA_EVENT_COMPLETE_CWR, /* end of congestion recovery */
Stephen Hemminger317a76f2005-06-23 12:19:55 -0700762 CA_EVENT_LOSS, /* loss timeout */
Florian Westphal98900922014-09-26 22:37:35 +0200763 CA_EVENT_ECN_NO_CE, /* ECT set, but not CE marked */
764 CA_EVENT_ECN_IS_CE, /* received CE marked IP packet */
765 CA_EVENT_DELAYED_ACK, /* Delayed ack is sent */
766 CA_EVENT_NON_DELAYED_ACK,
Florian Westphal7354c8c2014-09-26 22:37:34 +0200767};
768
Florian Westphal98900922014-09-26 22:37:35 +0200769/* Information about inbound ACK, passed to cong_ops->in_ack_event() */
Florian Westphal7354c8c2014-09-26 22:37:34 +0200770enum tcp_ca_ack_event_flags {
Florian Westphal98900922014-09-26 22:37:35 +0200771 CA_ACK_SLOWPATH = (1 << 0), /* In slow path processing */
772 CA_ACK_WIN_UPDATE = (1 << 1), /* ACK updated window */
773 CA_ACK_ECE = (1 << 2), /* ECE bit is set on ack */
Stephen Hemminger317a76f2005-06-23 12:19:55 -0700774};
775
776/*
777 * Interface for adding new TCP congestion control handlers
778 */
779#define TCP_CA_NAME_MAX 16
Stephen Hemminger3ff825b2006-11-09 16:32:06 -0800780#define TCP_CA_MAX 128
781#define TCP_CA_BUF_MAX (TCP_CA_NAME_MAX*TCP_CA_MAX)
782
Daniel Borkmann30e502a2014-09-26 22:37:33 +0200783/* Algorithm can be set on socket without CAP_NET_ADMIN privileges */
Stephen Hemminger164891a2007-04-23 22:26:16 -0700784#define TCP_CONG_NON_RESTRICTED 0x1
Daniel Borkmann30e502a2014-09-26 22:37:33 +0200785/* Requires ECN/ECT set on all packets */
786#define TCP_CONG_NEEDS_ECN 0x2
Stephen Hemminger164891a2007-04-23 22:26:16 -0700787
Stephen Hemminger317a76f2005-06-23 12:19:55 -0700788struct tcp_congestion_ops {
789 struct list_head list;
Stephen Hemminger164891a2007-04-23 22:26:16 -0700790 unsigned long flags;
Stephen Hemminger317a76f2005-06-23 12:19:55 -0700791
792 /* initialize private data (optional) */
Arnaldo Carvalho de Melo6687e982005-08-10 04:03:31 -0300793 void (*init)(struct sock *sk);
Stephen Hemminger317a76f2005-06-23 12:19:55 -0700794 /* cleanup private data (optional) */
Arnaldo Carvalho de Melo6687e982005-08-10 04:03:31 -0300795 void (*release)(struct sock *sk);
Stephen Hemminger317a76f2005-06-23 12:19:55 -0700796
797 /* return slow start threshold (required) */
Arnaldo Carvalho de Melo6687e982005-08-10 04:03:31 -0300798 u32 (*ssthresh)(struct sock *sk);
Stephen Hemminger317a76f2005-06-23 12:19:55 -0700799 /* do new cwnd calculation (required) */
Eric Dumazet24901552014-05-02 21:18:05 -0700800 void (*cong_avoid)(struct sock *sk, u32 ack, u32 acked);
Stephen Hemminger317a76f2005-06-23 12:19:55 -0700801 /* call before changing ca_state (optional) */
Arnaldo Carvalho de Melo6687e982005-08-10 04:03:31 -0300802 void (*set_state)(struct sock *sk, u8 new_state);
Stephen Hemminger317a76f2005-06-23 12:19:55 -0700803 /* call when cwnd event occurs (optional) */
Arnaldo Carvalho de Melo6687e982005-08-10 04:03:31 -0300804 void (*cwnd_event)(struct sock *sk, enum tcp_ca_event ev);
Florian Westphal7354c8c2014-09-26 22:37:34 +0200805 /* call when ack arrives (optional) */
806 void (*in_ack_event)(struct sock *sk, u32 flags);
Stephen Hemminger317a76f2005-06-23 12:19:55 -0700807 /* new value of cwnd after loss (optional) */
Arnaldo Carvalho de Melo6687e982005-08-10 04:03:31 -0300808 u32 (*undo_cwnd)(struct sock *sk);
Stephen Hemminger317a76f2005-06-23 12:19:55 -0700809 /* hook for packet ack accounting (optional) */
Stephen Hemminger30cfd0b2007-07-25 23:49:34 -0700810 void (*pkts_acked)(struct sock *sk, u32 num_acked, s32 rtt_us);
Arnaldo Carvalho de Melo73c1f4a2005-08-12 12:51:49 -0300811 /* get info for inet_diag (optional) */
Arnaldo Carvalho de Melo6687e982005-08-10 04:03:31 -0300812 void (*get_info)(struct sock *sk, u32 ext, struct sk_buff *skb);
Stephen Hemminger317a76f2005-06-23 12:19:55 -0700813
814 char name[TCP_CA_NAME_MAX];
815 struct module *owner;
816};
817
Joe Perches5c9f3022013-09-23 11:33:32 -0700818int tcp_register_congestion_control(struct tcp_congestion_ops *type);
819void tcp_unregister_congestion_control(struct tcp_congestion_ops *type);
Stephen Hemminger317a76f2005-06-23 12:19:55 -0700820
Florian Westphal55d86942014-09-26 22:37:32 +0200821void tcp_assign_congestion_control(struct sock *sk);
Joe Perches5c9f3022013-09-23 11:33:32 -0700822void tcp_init_congestion_control(struct sock *sk);
823void tcp_cleanup_congestion_control(struct sock *sk);
824int tcp_set_default_congestion_control(const char *name);
825void tcp_get_default_congestion_control(char *name);
826void tcp_get_available_congestion_control(char *buf, size_t len);
827void tcp_get_allowed_congestion_control(char *buf, size_t len);
828int tcp_set_allowed_congestion_control(char *allowed);
829int tcp_set_congestion_control(struct sock *sk, const char *name);
Li RongQinga12a6012014-09-30 09:49:55 +0800830void tcp_slow_start(struct tcp_sock *tp, u32 acked);
Joe Perches5c9f3022013-09-23 11:33:32 -0700831void tcp_cong_avoid_ai(struct tcp_sock *tp, u32 w);
Stephen Hemminger317a76f2005-06-23 12:19:55 -0700832
Joe Perches5c9f3022013-09-23 11:33:32 -0700833u32 tcp_reno_ssthresh(struct sock *sk);
Eric Dumazet24901552014-05-02 21:18:05 -0700834void tcp_reno_cong_avoid(struct sock *sk, u32 ack, u32 acked);
David S. Millera8acfba2005-06-23 23:45:02 -0700835extern struct tcp_congestion_ops tcp_reno;
Stephen Hemminger317a76f2005-06-23 12:19:55 -0700836
Daniel Borkmann30e502a2014-09-26 22:37:33 +0200837static inline bool tcp_ca_needs_ecn(const struct sock *sk)
838{
839 const struct inet_connection_sock *icsk = inet_csk(sk);
840
841 return icsk->icsk_ca_ops->flags & TCP_CONG_NEEDS_ECN;
842}
843
Arnaldo Carvalho de Melo6687e982005-08-10 04:03:31 -0300844static inline void tcp_set_ca_state(struct sock *sk, const u8 ca_state)
Stephen Hemminger317a76f2005-06-23 12:19:55 -0700845{
Arnaldo Carvalho de Melo6687e982005-08-10 04:03:31 -0300846 struct inet_connection_sock *icsk = inet_csk(sk);
847
848 if (icsk->icsk_ca_ops->set_state)
849 icsk->icsk_ca_ops->set_state(sk, ca_state);
850 icsk->icsk_ca_state = ca_state;
Stephen Hemminger317a76f2005-06-23 12:19:55 -0700851}
852
Arnaldo Carvalho de Melo6687e982005-08-10 04:03:31 -0300853static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event)
Stephen Hemminger317a76f2005-06-23 12:19:55 -0700854{
Arnaldo Carvalho de Melo6687e982005-08-10 04:03:31 -0300855 const struct inet_connection_sock *icsk = inet_csk(sk);
856
857 if (icsk->icsk_ca_ops->cwnd_event)
858 icsk->icsk_ca_ops->cwnd_event(sk, event);
Stephen Hemminger317a76f2005-06-23 12:19:55 -0700859}
860
Ilpo Järvinene60402d2007-08-09 15:14:46 +0300861/* These functions determine how the current flow behaves in respect of SACK
862 * handling. SACK is negotiated with the peer, and therefore it can vary
863 * between different flows.
864 *
865 * tcp_is_sack - SACK enabled
866 * tcp_is_reno - No SACK
867 * tcp_is_fack - FACK enabled, implies SACK enabled
868 */
869static inline int tcp_is_sack(const struct tcp_sock *tp)
870{
871 return tp->rx_opt.sack_ok;
872}
873
Eric Dumazeta2a385d2012-05-16 23:15:34 +0000874static inline bool tcp_is_reno(const struct tcp_sock *tp)
Ilpo Järvinene60402d2007-08-09 15:14:46 +0300875{
876 return !tcp_is_sack(tp);
877}
878
Eric Dumazeta2a385d2012-05-16 23:15:34 +0000879static inline bool tcp_is_fack(const struct tcp_sock *tp)
Ilpo Järvinene60402d2007-08-09 15:14:46 +0300880{
Vijay Subramanianab562222011-12-20 13:23:24 +0000881 return tp->rx_opt.sack_ok & TCP_FACK_ENABLED;
Ilpo Järvinene60402d2007-08-09 15:14:46 +0300882}
883
884static inline void tcp_enable_fack(struct tcp_sock *tp)
885{
Vijay Subramanianab562222011-12-20 13:23:24 +0000886 tp->rx_opt.sack_ok |= TCP_FACK_ENABLED;
Ilpo Järvinene60402d2007-08-09 15:14:46 +0300887}
888
Yuchung Chengeed530b2012-05-02 13:30:03 +0000889/* TCP early-retransmit (ER) is similar to but more conservative than
890 * the thin-dupack feature. Enable ER only if thin-dupack is disabled.
891 */
892static inline void tcp_enable_early_retrans(struct tcp_sock *tp)
893{
894 tp->do_early_retrans = sysctl_tcp_early_retrans &&
Nandita Dukkipati6ba8a3b2013-03-11 10:00:43 +0000895 sysctl_tcp_early_retrans < 4 && !sysctl_tcp_thin_dupack &&
896 sysctl_tcp_reordering == 3;
Yuchung Chengeed530b2012-05-02 13:30:03 +0000897}
898
899static inline void tcp_disable_early_retrans(struct tcp_sock *tp)
900{
901 tp->do_early_retrans = 0;
902}
903
Ilpo Järvinen83ae4082007-08-09 14:37:30 +0300904static inline unsigned int tcp_left_out(const struct tcp_sock *tp)
905{
906 return tp->sacked_out + tp->lost_out;
907}
908
Linus Torvalds1da177e2005-04-16 15:20:36 -0700909/* This determines how many packets are "in the network" to the best
910 * of our knowledge. In many cases it is conservative, but where
911 * detailed information is available from the receiver (via SACK
912 * blocks etc.) we can make more aggressive calculations.
913 *
914 * Use this for decisions involving congestion control, use just
915 * tp->packets_out to determine if the send queue is empty or not.
916 *
917 * Read this equation as:
918 *
919 * "Packets sent once on transmission queue" MINUS
920 * "Packets left network, but not honestly ACKed yet" PLUS
921 * "Packets fast retransmitted"
922 */
Stephen Hemminger40efc6f2006-01-03 16:03:49 -0800923static inline unsigned int tcp_packets_in_flight(const struct tcp_sock *tp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700924{
Ilpo Järvinen83ae4082007-08-09 14:37:30 +0300925 return tp->packets_out - tcp_left_out(tp) + tp->retrans_out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700926}
927
Ilpo Järvinen0b6a05c2009-09-15 01:30:10 -0700928#define TCP_INFINITE_SSTHRESH 0x7fffffff
929
930static inline bool tcp_in_initial_slowstart(const struct tcp_sock *tp)
931{
932 return tp->snd_ssthresh >= TCP_INFINITE_SSTHRESH;
933}
934
Yuchung Cheng684bad12012-09-02 17:38:04 +0000935static inline bool tcp_in_cwnd_reduction(const struct sock *sk)
936{
937 return (TCPF_CA_CWR | TCPF_CA_Recovery) &
938 (1 << inet_csk(sk)->icsk_ca_state);
939}
940
Linus Torvalds1da177e2005-04-16 15:20:36 -0700941/* If cwnd > ssthresh, we may raise ssthresh to be half-way to cwnd.
Yuchung Cheng684bad12012-09-02 17:38:04 +0000942 * The exception is cwnd reduction phase, when cwnd is decreasing towards
Linus Torvalds1da177e2005-04-16 15:20:36 -0700943 * ssthresh.
944 */
Arnaldo Carvalho de Melo6687e982005-08-10 04:03:31 -0300945static inline __u32 tcp_current_ssthresh(const struct sock *sk)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700946{
Arnaldo Carvalho de Melo6687e982005-08-10 04:03:31 -0300947 const struct tcp_sock *tp = tcp_sk(sk);
Eric Dumazetcf533ea2011-10-21 05:22:42 -0400948
Yuchung Cheng684bad12012-09-02 17:38:04 +0000949 if (tcp_in_cwnd_reduction(sk))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700950 return tp->snd_ssthresh;
951 else
952 return max(tp->snd_ssthresh,
953 ((tp->snd_cwnd >> 1) +
954 (tp->snd_cwnd >> 2)));
955}
956
Ilpo Järvinenb9c45952007-07-27 16:36:17 +0300957/* Use define here intentionally to get WARN_ON location shown at the caller */
958#define tcp_verify_left_out(tp) WARN_ON(tcp_left_out(tp) > tp->packets_out)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700959
Christoph Paasch5ee2c942014-07-14 16:58:32 +0200960void tcp_enter_cwr(struct sock *sk);
Joe Perches5c9f3022013-09-23 11:33:32 -0700961__u32 tcp_init_cwnd(const struct tcp_sock *tp, const struct dst_entry *dst);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700962
Neal Cardwell6b5a5c02011-11-21 17:15:14 +0000963/* The maximum number of MSS of available cwnd for which TSO defers
964 * sending if not using sysctl_tcp_tso_win_divisor.
965 */
966static inline __u32 tcp_max_tso_deferred_mss(const struct tcp_sock *tp)
967{
968 return 3;
969}
970
Linus Torvalds1da177e2005-04-16 15:20:36 -0700971/* Slow start with delack produces 3 packets of burst, so that
John Heffnerdd9e0dd2008-04-15 15:26:39 -0700972 * it is safe "de facto". This will be the default - same as
973 * the default reordering threshold - but if reordering increases,
974 * we must be able to allow cwnd to burst at least this much in order
975 * to not pull it back when holes are filled.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700976 */
977static __inline__ __u32 tcp_max_burst(const struct tcp_sock *tp)
978{
John Heffnerdd9e0dd2008-04-15 15:26:39 -0700979 return tp->reordering;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700980}
981
Ilpo Järvinen90840de2007-12-31 04:48:41 -0800982/* Returns end sequence number of the receiver's advertised window */
983static inline u32 tcp_wnd_end(const struct tcp_sock *tp)
984{
985 return tp->snd_una + tp->snd_wnd;
986}
Eric Dumazete114a712014-04-30 11:58:13 -0700987
988/* We follow the spirit of RFC2861 to validate cwnd but implement a more
989 * flexible approach. The RFC suggests cwnd should not be raised unless
Neal Cardwellca8a2262014-05-22 10:41:08 -0400990 * it was fully used previously. And that's exactly what we do in
991 * congestion avoidance mode. But in slow start we allow cwnd to grow
992 * as long as the application has used half the cwnd.
Eric Dumazete114a712014-04-30 11:58:13 -0700993 * Example :
994 * cwnd is 10 (IW10), but application sends 9 frames.
995 * We allow cwnd to reach 18 when all frames are ACKed.
996 * This check is safe because it's as aggressive as slow start which already
997 * risks 100% overshoot. The advantage is that we discourage application to
998 * either send more filler packets or data to artificially blow up the cwnd
999 * usage, and allow application-limited process to probe bw more aggressively.
Eric Dumazete114a712014-04-30 11:58:13 -07001000 */
Eric Dumazet24901552014-05-02 21:18:05 -07001001static inline bool tcp_is_cwnd_limited(const struct sock *sk)
Eric Dumazete114a712014-04-30 11:58:13 -07001002{
1003 const struct tcp_sock *tp = tcp_sk(sk);
1004
Neal Cardwellca8a2262014-05-22 10:41:08 -04001005 /* If in slow start, ensure cwnd grows to twice what was ACKed. */
1006 if (tp->snd_cwnd <= tp->snd_ssthresh)
1007 return tp->snd_cwnd < 2 * tp->max_packets_out;
1008
1009 return tp->is_cwnd_limited;
Eric Dumazete114a712014-04-30 11:58:13 -07001010}
Stephen Hemmingerf4805ed2005-11-10 16:53:30 -08001011
Ilpo Järvinen9e412ba2007-04-20 22:18:02 -07001012static inline void tcp_check_probe_timer(struct sock *sk)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001013{
Eric Dumazetcf533ea2011-10-21 05:22:42 -04001014 const struct tcp_sock *tp = tcp_sk(sk);
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -07001015 const struct inet_connection_sock *icsk = inet_csk(sk);
Ilpo Järvinen9e412ba2007-04-20 22:18:02 -07001016
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -07001017 if (!tp->packets_out && !icsk->icsk_pending)
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -07001018 inet_csk_reset_xmit_timer(sk, ICSK_TIME_PROBE0,
1019 icsk->icsk_rto, TCP_RTO_MAX);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001020}
1021
Hantzis Fotisee7537b2009-03-02 22:42:02 -08001022static inline void tcp_init_wl(struct tcp_sock *tp, u32 seq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001023{
1024 tp->snd_wl1 = seq;
1025}
1026
Hantzis Fotisee7537b2009-03-02 22:42:02 -08001027static inline void tcp_update_wl(struct tcp_sock *tp, u32 seq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001028{
1029 tp->snd_wl1 = seq;
1030}
1031
Linus Torvalds1da177e2005-04-16 15:20:36 -07001032/*
1033 * Calculate(/check) TCP checksum
1034 */
Frederik Deweerdtba7808e2007-02-04 20:15:27 -08001035static inline __sum16 tcp_v4_check(int len, __be32 saddr,
1036 __be32 daddr, __wsum base)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001037{
1038 return csum_tcpudp_magic(saddr,daddr,len,IPPROTO_TCP,base);
1039}
1040
Al Virob51655b2006-11-14 21:40:42 -08001041static inline __sum16 __tcp_checksum_complete(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001042{
Herbert Xufb286bb2005-11-10 13:01:24 -08001043 return __skb_checksum_complete(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001044}
1045
Eric Dumazeta2a385d2012-05-16 23:15:34 +00001046static inline bool tcp_checksum_complete(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001047{
Herbert Xu60476372007-04-09 11:59:39 -07001048 return !skb_csum_unnecessary(skb) &&
Linus Torvalds1da177e2005-04-16 15:20:36 -07001049 __tcp_checksum_complete(skb);
1050}
1051
1052/* Prequeue for VJ style copy to user, combined with checksumming. */
1053
Stephen Hemminger40efc6f2006-01-03 16:03:49 -08001054static inline void tcp_prequeue_init(struct tcp_sock *tp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001055{
1056 tp->ucopy.task = NULL;
1057 tp->ucopy.len = 0;
1058 tp->ucopy.memory = 0;
1059 skb_queue_head_init(&tp->ucopy.prequeue);
1060}
1061
Joe Perches5c9f3022013-09-23 11:33:32 -07001062bool tcp_prequeue(struct sock *sk, struct sk_buff *skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001063
1064#undef STATE_TRACE
1065
1066#ifdef STATE_TRACE
1067static const char *statename[]={
1068 "Unused","Established","Syn Sent","Syn Recv",
1069 "Fin Wait 1","Fin Wait 2","Time Wait", "Close",
1070 "Close Wait","Last ACK","Listen","Closing"
1071};
1072#endif
Joe Perches5c9f3022013-09-23 11:33:32 -07001073void tcp_set_state(struct sock *sk, int state);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001074
Joe Perches5c9f3022013-09-23 11:33:32 -07001075void tcp_done(struct sock *sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001076
Stephen Hemminger40efc6f2006-01-03 16:03:49 -08001077static inline void tcp_sack_reset(struct tcp_options_received *rx_opt)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001078{
1079 rx_opt->dsack = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001080 rx_opt->num_sacks = 0;
1081}
1082
Joe Perches5c9f3022013-09-23 11:33:32 -07001083u32 tcp_default_init_rwnd(u32 mss);
Yuchung Cheng85f16522013-06-11 15:35:32 -07001084
Linus Torvalds1da177e2005-04-16 15:20:36 -07001085/* Determine a window scaling and initial window to offer. */
Joe Perches5c9f3022013-09-23 11:33:32 -07001086void tcp_select_initial_window(int __space, __u32 mss, __u32 *rcv_wnd,
1087 __u32 *window_clamp, int wscale_ok,
1088 __u8 *rcv_wscale, __u32 init_rcv_wnd);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001089
1090static inline int tcp_win_from_space(int space)
1091{
1092 return sysctl_tcp_adv_win_scale<=0 ?
1093 (space>>(-sysctl_tcp_adv_win_scale)) :
1094 space - (space>>sysctl_tcp_adv_win_scale);
1095}
1096
1097/* Note: caller must be prepared to deal with negative returns */
1098static inline int tcp_space(const struct sock *sk)
1099{
1100 return tcp_win_from_space(sk->sk_rcvbuf -
1101 atomic_read(&sk->sk_rmem_alloc));
1102}
1103
1104static inline int tcp_full_space(const struct sock *sk)
1105{
1106 return tcp_win_from_space(sk->sk_rcvbuf);
1107}
1108
Stephen Hemminger40efc6f2006-01-03 16:03:49 -08001109static inline void tcp_openreq_init(struct request_sock *req,
1110 struct tcp_options_received *rx_opt,
Octavian Purdilae0f802f2014-06-17 11:25:37 +03001111 struct sk_buff *skb, struct sock *sk)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001112{
Arnaldo Carvalho de Melo2e6599c2005-06-18 22:46:52 -07001113 struct inet_request_sock *ireq = inet_rsk(req);
1114
Linus Torvalds1da177e2005-04-16 15:20:36 -07001115 req->rcv_wnd = 0; /* So that tcp_send_synack() knows! */
Florian Westphal4dfc2812008-04-10 03:12:40 -07001116 req->cookie_ts = 0;
Arnaldo Carvalho de Melo2e6599c2005-06-18 22:46:52 -07001117 tcp_rsk(req)->rcv_isn = TCP_SKB_CB(skb)->seq;
Jerry Chu10467162012-08-31 12:29:11 +00001118 tcp_rsk(req)->rcv_nxt = TCP_SKB_CB(skb)->seq + 1;
Neal Cardwell86c6a2c2014-06-30 15:09:49 -04001119 tcp_rsk(req)->snt_synack = tcp_time_stamp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001120 req->mss = rx_opt->mss_clamp;
1121 req->ts_recent = rx_opt->saw_tstamp ? rx_opt->rcv_tsval : 0;
Arnaldo Carvalho de Melo2e6599c2005-06-18 22:46:52 -07001122 ireq->tstamp_ok = rx_opt->tstamp_ok;
1123 ireq->sack_ok = rx_opt->sack_ok;
1124 ireq->snd_wscale = rx_opt->snd_wscale;
1125 ireq->wscale_ok = rx_opt->wscale_ok;
1126 ireq->acked = 0;
1127 ireq->ecn_ok = 0;
Eric Dumazet634fb9792013-10-09 15:21:29 -07001128 ireq->ir_rmt_port = tcp_hdr(skb)->source;
Eric Dumazetb44084c2013-10-10 00:04:37 -07001129 ireq->ir_num = ntohs(tcp_hdr(skb)->dest);
Octavian Purdilae0f802f2014-06-17 11:25:37 +03001130 ireq->ir_mark = inet_request_mark(sk, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001131}
1132
Yuchung Cheng843f4a52014-05-11 20:22:11 -07001133extern void tcp_openreq_init_rwin(struct request_sock *req,
1134 struct sock *sk, struct dst_entry *dst);
1135
Joe Perches5c9f3022013-09-23 11:33:32 -07001136void tcp_enter_memory_pressure(struct sock *sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001137
Linus Torvalds1da177e2005-04-16 15:20:36 -07001138static inline int keepalive_intvl_when(const struct tcp_sock *tp)
1139{
1140 return tp->keepalive_intvl ? : sysctl_tcp_keepalive_intvl;
1141}
1142
1143static inline int keepalive_time_when(const struct tcp_sock *tp)
1144{
1145 return tp->keepalive_time ? : sysctl_tcp_keepalive_time;
1146}
1147
Eric Dumazetdf19a622009-08-28 23:48:54 -07001148static inline int keepalive_probes(const struct tcp_sock *tp)
1149{
1150 return tp->keepalive_probes ? : sysctl_tcp_keepalive_probes;
1151}
1152
Flavio Leitner6c37e5d2010-04-26 18:33:27 +00001153static inline u32 keepalive_time_elapsed(const struct tcp_sock *tp)
1154{
1155 const struct inet_connection_sock *icsk = &tp->inet_conn;
1156
1157 return min_t(u32, tcp_time_stamp - icsk->icsk_ack.lrcvtime,
1158 tcp_time_stamp - tp->rcv_tstamp);
1159}
1160
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -07001161static inline int tcp_fin_time(const struct sock *sk)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001162{
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -07001163 int fin_timeout = tcp_sk(sk)->linger2 ? : sysctl_tcp_fin_timeout;
1164 const int rto = inet_csk(sk)->icsk_rto;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001165
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -07001166 if (fin_timeout < (rto << 2) - (rto >> 1))
1167 fin_timeout = (rto << 2) - (rto >> 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001168
1169 return fin_timeout;
1170}
1171
Eric Dumazeta2a385d2012-05-16 23:15:34 +00001172static inline bool tcp_paws_check(const struct tcp_options_received *rx_opt,
1173 int paws_win)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001174{
Ilpo Järvinenc887e6d2009-03-14 14:23:03 +00001175 if ((s32)(rx_opt->ts_recent - rx_opt->rcv_tsval) <= paws_win)
Eric Dumazeta2a385d2012-05-16 23:15:34 +00001176 return true;
Ilpo Järvinenc887e6d2009-03-14 14:23:03 +00001177 if (unlikely(get_seconds() >= rx_opt->ts_recent_stamp + TCP_PAWS_24DAYS))
Eric Dumazeta2a385d2012-05-16 23:15:34 +00001178 return true;
Eric Dumazetbc2ce892010-12-16 14:08:34 -08001179 /*
1180 * Some OSes send SYN and SYNACK messages with tsval=0 tsecr=0,
1181 * then following tcp messages have valid values. Ignore 0 value,
1182 * or else 'negative' tsval might forbid us to accept their packets.
1183 */
1184 if (!rx_opt->ts_recent)
Eric Dumazeta2a385d2012-05-16 23:15:34 +00001185 return true;
1186 return false;
Ilpo Järvinenc887e6d2009-03-14 14:23:03 +00001187}
1188
Eric Dumazeta2a385d2012-05-16 23:15:34 +00001189static inline bool tcp_paws_reject(const struct tcp_options_received *rx_opt,
1190 int rst)
Ilpo Järvinenc887e6d2009-03-14 14:23:03 +00001191{
1192 if (tcp_paws_check(rx_opt, 0))
Eric Dumazeta2a385d2012-05-16 23:15:34 +00001193 return false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001194
1195 /* RST segments are not recommended to carry timestamp,
1196 and, if they do, it is recommended to ignore PAWS because
1197 "their cleanup function should take precedence over timestamps."
1198 Certainly, it is mistake. It is necessary to understand the reasons
1199 of this constraint to relax it: if peer reboots, clock may go
1200 out-of-sync and half-open connections will not be reset.
1201 Actually, the problem would be not existing if all
1202 the implementations followed draft about maintaining clock
1203 via reboots. Linux-2.2 DOES NOT!
1204
1205 However, we can relax time bounds for RST segments to MSL.
1206 */
James Morris9d729f72007-03-04 16:12:44 -08001207 if (rst && get_seconds() >= rx_opt->ts_recent_stamp + TCP_PAWS_MSL)
Eric Dumazeta2a385d2012-05-16 23:15:34 +00001208 return false;
1209 return true;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001210}
1211
Pavel Emelyanova9c193292008-07-16 20:21:42 -07001212static inline void tcp_mib_init(struct net *net)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001213{
1214 /* See RFC 2012 */
Pavel Emelyanovcf1100a2008-07-16 20:27:38 -07001215 TCP_ADD_STATS_USER(net, TCP_MIB_RTOALGORITHM, 1);
1216 TCP_ADD_STATS_USER(net, TCP_MIB_RTOMIN, TCP_RTO_MIN*1000/HZ);
1217 TCP_ADD_STATS_USER(net, TCP_MIB_RTOMAX, TCP_RTO_MAX*1000/HZ);
1218 TCP_ADD_STATS_USER(net, TCP_MIB_MAXCONN, -1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001219}
1220
Ilpo Järvinen5af4ec22007-09-20 11:30:48 -07001221/* from STCP */
Ilpo Järvinenef9da472008-09-20 21:25:15 -07001222static inline void tcp_clear_retrans_hints_partial(struct tcp_sock *tp)
David S. Miller0800f172007-09-20 11:40:37 -07001223{
Stephen Hemminger6a438bb2005-11-10 17:14:59 -08001224 tp->lost_skb_hint = NULL;
Ilpo Järvinenef9da472008-09-20 21:25:15 -07001225}
1226
1227static inline void tcp_clear_all_retrans_hints(struct tcp_sock *tp)
1228{
1229 tcp_clear_retrans_hints_partial(tp);
Stephen Hemminger6a438bb2005-11-10 17:14:59 -08001230 tp->retransmit_skb_hint = NULL;
Ilpo Järvinenb7689202007-09-20 11:37:19 -07001231}
1232
YOSHIFUJI Hideakicfb6eeb2006-11-14 19:07:45 -08001233/* MD5 Signature */
1234struct crypto_hash;
1235
Eric Dumazeta915da9b2012-01-31 05:18:33 +00001236union tcp_md5_addr {
1237 struct in_addr a4;
1238#if IS_ENABLED(CONFIG_IPV6)
1239 struct in6_addr a6;
1240#endif
1241};
1242
YOSHIFUJI Hideakicfb6eeb2006-11-14 19:07:45 -08001243/* - key database */
1244struct tcp_md5sig_key {
Eric Dumazeta915da9b2012-01-31 05:18:33 +00001245 struct hlist_node node;
YOSHIFUJI Hideakicfb6eeb2006-11-14 19:07:45 -08001246 u8 keylen;
Eric Dumazeta915da9b2012-01-31 05:18:33 +00001247 u8 family; /* AF_INET or AF_INET6 */
1248 union tcp_md5_addr addr;
1249 u8 key[TCP_MD5SIG_MAXKEYLEN];
1250 struct rcu_head rcu;
YOSHIFUJI Hideakicfb6eeb2006-11-14 19:07:45 -08001251};
1252
1253/* - sock block */
1254struct tcp_md5sig_info {
Eric Dumazeta915da9b2012-01-31 05:18:33 +00001255 struct hlist_head head;
Eric Dumazeta8afca02012-01-31 18:45:40 +00001256 struct rcu_head rcu;
YOSHIFUJI Hideakicfb6eeb2006-11-14 19:07:45 -08001257};
1258
1259/* - pseudo header */
1260struct tcp4_pseudohdr {
1261 __be32 saddr;
1262 __be32 daddr;
1263 __u8 pad;
1264 __u8 protocol;
1265 __be16 len;
1266};
1267
1268struct tcp6_pseudohdr {
1269 struct in6_addr saddr;
1270 struct in6_addr daddr;
1271 __be32 len;
1272 __be32 protocol; /* including padding */
1273};
1274
1275union tcp_md5sum_block {
1276 struct tcp4_pseudohdr ip4;
Eric Dumazetdfd56b82011-12-10 09:48:31 +00001277#if IS_ENABLED(CONFIG_IPV6)
YOSHIFUJI Hideakicfb6eeb2006-11-14 19:07:45 -08001278 struct tcp6_pseudohdr ip6;
1279#endif
1280};
1281
1282/* - pool: digest algorithm, hash description and scratch buffer */
1283struct tcp_md5sig_pool {
1284 struct hash_desc md5_desc;
1285 union tcp_md5sum_block md5_blk;
1286};
1287
YOSHIFUJI Hideakicfb6eeb2006-11-14 19:07:45 -08001288/* - functions */
Joe Perches5c9f3022013-09-23 11:33:32 -07001289int tcp_v4_md5_hash_skb(char *md5_hash, struct tcp_md5sig_key *key,
1290 const struct sock *sk, const struct request_sock *req,
1291 const struct sk_buff *skb);
1292int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr,
1293 int family, const u8 *newkey, u8 newkeylen, gfp_t gfp);
1294int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr,
1295 int family);
1296struct tcp_md5sig_key *tcp_v4_md5_lookup(struct sock *sk,
Eric Dumazeta915da9b2012-01-31 05:18:33 +00001297 struct sock *addr_sk);
YOSHIFUJI Hideakicfb6eeb2006-11-14 19:07:45 -08001298
YOSHIFUJI Hideaki9501f972008-04-18 12:45:16 +09001299#ifdef CONFIG_TCP_MD5SIG
Joe Perches5c9f3022013-09-23 11:33:32 -07001300struct tcp_md5sig_key *tcp_md5_do_lookup(struct sock *sk,
1301 const union tcp_md5_addr *addr,
1302 int family);
Eric Dumazeta915da9b2012-01-31 05:18:33 +00001303#define tcp_twsk_md5_key(twsk) ((twsk)->tw_md5_key)
YOSHIFUJI Hideaki9501f972008-04-18 12:45:16 +09001304#else
Eric Dumazeta915da9b2012-01-31 05:18:33 +00001305static inline struct tcp_md5sig_key *tcp_md5_do_lookup(struct sock *sk,
1306 const union tcp_md5_addr *addr,
1307 int family)
1308{
1309 return NULL;
1310}
YOSHIFUJI Hideaki9501f972008-04-18 12:45:16 +09001311#define tcp_twsk_md5_key(twsk) NULL
1312#endif
1313
Joe Perches5c9f3022013-09-23 11:33:32 -07001314bool tcp_alloc_md5sig_pool(void);
YOSHIFUJI Hideakicfb6eeb2006-11-14 19:07:45 -08001315
Joe Perches5c9f3022013-09-23 11:33:32 -07001316struct tcp_md5sig_pool *tcp_get_md5sig_pool(void);
Eric Dumazet71cea172013-05-20 06:52:26 +00001317static inline void tcp_put_md5sig_pool(void)
1318{
1319 local_bh_enable();
1320}
Eric Dumazet35790c02010-05-16 00:34:04 -07001321
Joe Perches5c9f3022013-09-23 11:33:32 -07001322int tcp_md5_hash_header(struct tcp_md5sig_pool *, const struct tcphdr *);
1323int tcp_md5_hash_skb_data(struct tcp_md5sig_pool *, const struct sk_buff *,
1324 unsigned int header_len);
1325int tcp_md5_hash_key(struct tcp_md5sig_pool *hp,
1326 const struct tcp_md5sig_key *key);
YOSHIFUJI Hideakicfb6eeb2006-11-14 19:07:45 -08001327
Jerry Chu10467162012-08-31 12:29:11 +00001328/* From tcp_fastopen.c */
Joe Perches5c9f3022013-09-23 11:33:32 -07001329void tcp_fastopen_cache_get(struct sock *sk, u16 *mss,
1330 struct tcp_fastopen_cookie *cookie, int *syn_loss,
1331 unsigned long *last_syn_loss);
1332void tcp_fastopen_cache_set(struct sock *sk, u16 mss,
1333 struct tcp_fastopen_cookie *cookie, bool syn_lost);
Yuchung Cheng783237e2012-07-19 06:43:07 +00001334struct tcp_fastopen_request {
1335 /* Fast Open cookie. Size 0 means a cookie request */
1336 struct tcp_fastopen_cookie cookie;
1337 struct msghdr *data; /* data in MSG_FASTOPEN */
Eric Dumazetf5ddcbb2014-02-20 10:09:18 -08001338 size_t size;
1339 int copied; /* queued in tcp_connect() */
Yuchung Cheng783237e2012-07-19 06:43:07 +00001340};
Yuchung Cheng783237e2012-07-19 06:43:07 +00001341void tcp_free_fastopen_req(struct tcp_sock *tp);
1342
Jerry Chu10467162012-08-31 12:29:11 +00001343extern struct tcp_fastopen_context __rcu *tcp_fastopen_ctx;
1344int tcp_fastopen_reset_cipher(void *key, unsigned int len);
Yuchung Cheng843f4a52014-05-11 20:22:11 -07001345bool tcp_try_fastopen(struct sock *sk, struct sk_buff *skb,
1346 struct request_sock *req,
1347 struct tcp_fastopen_cookie *foc,
1348 struct dst_entry *dst);
Hannes Frederic Sowa222e83d2013-10-19 21:48:58 +02001349void tcp_fastopen_init_key_once(bool publish);
Jerry Chu10467162012-08-31 12:29:11 +00001350#define TCP_FASTOPEN_KEY_LENGTH 16
1351
1352/* Fastopen key context */
1353struct tcp_fastopen_context {
Eric Dumazet7ae86392013-06-25 01:21:06 -07001354 struct crypto_cipher *tfm;
1355 __u8 key[TCP_FASTOPEN_KEY_LENGTH];
1356 struct rcu_head rcu;
Jerry Chu10467162012-08-31 12:29:11 +00001357};
1358
David S. Millerfe067e82007-03-07 12:12:44 -08001359/* write queue abstraction */
1360static inline void tcp_write_queue_purge(struct sock *sk)
1361{
1362 struct sk_buff *skb;
1363
1364 while ((skb = __skb_dequeue(&sk->sk_write_queue)) != NULL)
Hideo Aoki3ab224b2007-12-31 00:11:19 -08001365 sk_wmem_free_skb(sk, skb);
1366 sk_mem_reclaim(sk);
Ilpo Järvinen8818a9d2009-12-02 22:24:02 -08001367 tcp_clear_all_retrans_hints(tcp_sk(sk));
David S. Millerfe067e82007-03-07 12:12:44 -08001368}
1369
Eric Dumazetcf533ea2011-10-21 05:22:42 -04001370static inline struct sk_buff *tcp_write_queue_head(const struct sock *sk)
David S. Millerfe067e82007-03-07 12:12:44 -08001371{
David S. Millercd07a8e2008-09-23 00:50:13 -07001372 return skb_peek(&sk->sk_write_queue);
David S. Millerfe067e82007-03-07 12:12:44 -08001373}
1374
Eric Dumazetcf533ea2011-10-21 05:22:42 -04001375static inline struct sk_buff *tcp_write_queue_tail(const struct sock *sk)
David S. Millerfe067e82007-03-07 12:12:44 -08001376{
David S. Millercd07a8e2008-09-23 00:50:13 -07001377 return skb_peek_tail(&sk->sk_write_queue);
David S. Millerfe067e82007-03-07 12:12:44 -08001378}
1379
Eric Dumazetcf533ea2011-10-21 05:22:42 -04001380static inline struct sk_buff *tcp_write_queue_next(const struct sock *sk,
1381 const struct sk_buff *skb)
David S. Millerfe067e82007-03-07 12:12:44 -08001382{
David S. Millercd07a8e2008-09-23 00:50:13 -07001383 return skb_queue_next(&sk->sk_write_queue, skb);
David S. Millerfe067e82007-03-07 12:12:44 -08001384}
1385
Eric Dumazetcf533ea2011-10-21 05:22:42 -04001386static inline struct sk_buff *tcp_write_queue_prev(const struct sock *sk,
1387 const struct sk_buff *skb)
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08001388{
1389 return skb_queue_prev(&sk->sk_write_queue, skb);
1390}
1391
David S. Millerfe067e82007-03-07 12:12:44 -08001392#define tcp_for_write_queue(skb, sk) \
David S. Millercd07a8e2008-09-23 00:50:13 -07001393 skb_queue_walk(&(sk)->sk_write_queue, skb)
David S. Millerfe067e82007-03-07 12:12:44 -08001394
1395#define tcp_for_write_queue_from(skb, sk) \
David S. Millercd07a8e2008-09-23 00:50:13 -07001396 skb_queue_walk_from(&(sk)->sk_write_queue, skb)
David S. Millerfe067e82007-03-07 12:12:44 -08001397
Ilpo Järvinen234b6862007-12-02 00:48:02 +02001398#define tcp_for_write_queue_from_safe(skb, tmp, sk) \
David S. Millercd07a8e2008-09-23 00:50:13 -07001399 skb_queue_walk_from_safe(&(sk)->sk_write_queue, skb, tmp)
Ilpo Järvinen234b6862007-12-02 00:48:02 +02001400
Eric Dumazetcf533ea2011-10-21 05:22:42 -04001401static inline struct sk_buff *tcp_send_head(const struct sock *sk)
David S. Millerfe067e82007-03-07 12:12:44 -08001402{
1403 return sk->sk_send_head;
1404}
1405
David S. Millercd07a8e2008-09-23 00:50:13 -07001406static inline bool tcp_skb_is_last(const struct sock *sk,
1407 const struct sk_buff *skb)
1408{
1409 return skb_queue_is_last(&sk->sk_write_queue, skb);
1410}
1411
Eric Dumazetcf533ea2011-10-21 05:22:42 -04001412static inline void tcp_advance_send_head(struct sock *sk, const struct sk_buff *skb)
David S. Millerfe067e82007-03-07 12:12:44 -08001413{
David S. Millercd07a8e2008-09-23 00:50:13 -07001414 if (tcp_skb_is_last(sk, skb))
David S. Millerfe067e82007-03-07 12:12:44 -08001415 sk->sk_send_head = NULL;
David S. Millercd07a8e2008-09-23 00:50:13 -07001416 else
1417 sk->sk_send_head = tcp_write_queue_next(sk, skb);
David S. Millerfe067e82007-03-07 12:12:44 -08001418}
1419
1420static inline void tcp_check_send_head(struct sock *sk, struct sk_buff *skb_unlinked)
1421{
1422 if (sk->sk_send_head == skb_unlinked)
1423 sk->sk_send_head = NULL;
1424}
1425
1426static inline void tcp_init_send_head(struct sock *sk)
1427{
1428 sk->sk_send_head = NULL;
1429}
1430
1431static inline void __tcp_add_write_queue_tail(struct sock *sk, struct sk_buff *skb)
1432{
1433 __skb_queue_tail(&sk->sk_write_queue, skb);
1434}
1435
1436static inline void tcp_add_write_queue_tail(struct sock *sk, struct sk_buff *skb)
1437{
1438 __tcp_add_write_queue_tail(sk, skb);
1439
1440 /* Queue it, remembering where we must start sending. */
Ilpo Järvinen6859d492007-12-02 00:48:06 +02001441 if (sk->sk_send_head == NULL) {
David S. Millerfe067e82007-03-07 12:12:44 -08001442 sk->sk_send_head = skb;
Ilpo Järvinen6859d492007-12-02 00:48:06 +02001443
1444 if (tcp_sk(sk)->highest_sack == NULL)
1445 tcp_sk(sk)->highest_sack = skb;
1446 }
David S. Millerfe067e82007-03-07 12:12:44 -08001447}
1448
1449static inline void __tcp_add_write_queue_head(struct sock *sk, struct sk_buff *skb)
1450{
1451 __skb_queue_head(&sk->sk_write_queue, skb);
1452}
1453
1454/* Insert buff after skb on the write queue of sk. */
1455static inline void tcp_insert_write_queue_after(struct sk_buff *skb,
1456 struct sk_buff *buff,
1457 struct sock *sk)
1458{
Gerrit Renker7de6c032008-04-14 00:05:09 -07001459 __skb_queue_after(&sk->sk_write_queue, skb, buff);
David S. Millerfe067e82007-03-07 12:12:44 -08001460}
1461
David S. Miller43f59c82008-09-21 21:28:51 -07001462/* Insert new before skb on the write queue of sk. */
David S. Millerfe067e82007-03-07 12:12:44 -08001463static inline void tcp_insert_write_queue_before(struct sk_buff *new,
1464 struct sk_buff *skb,
1465 struct sock *sk)
1466{
David S. Miller43f59c82008-09-21 21:28:51 -07001467 __skb_queue_before(&sk->sk_write_queue, skb, new);
Ilpo Järvinen6e421412007-11-19 23:24:09 -08001468
1469 if (sk->sk_send_head == skb)
1470 sk->sk_send_head = new;
David S. Millerfe067e82007-03-07 12:12:44 -08001471}
1472
1473static inline void tcp_unlink_write_queue(struct sk_buff *skb, struct sock *sk)
1474{
1475 __skb_unlink(skb, &sk->sk_write_queue);
1476}
1477
Eric Dumazeta2a385d2012-05-16 23:15:34 +00001478static inline bool tcp_write_queue_empty(struct sock *sk)
David S. Millerfe067e82007-03-07 12:12:44 -08001479{
1480 return skb_queue_empty(&sk->sk_write_queue);
1481}
1482
Krishna Kumar12d50c42009-12-08 22:26:13 +00001483static inline void tcp_push_pending_frames(struct sock *sk)
1484{
1485 if (tcp_send_head(sk)) {
1486 struct tcp_sock *tp = tcp_sk(sk);
1487
1488 __tcp_push_pending_frames(sk, tcp_current_mss(sk), tp->nonagle);
1489 }
1490}
1491
Neal Cardwellecb97192012-02-27 17:52:52 -05001492/* Start sequence of the skb just after the highest skb with SACKed
1493 * bit, valid only if sacked_out > 0 or when the caller has ensured
1494 * validity by itself.
Ilpo Järvinena47e5a92007-11-15 19:41:46 -08001495 */
1496static inline u32 tcp_highest_sack_seq(struct tcp_sock *tp)
1497{
1498 if (!tp->sacked_out)
1499 return tp->snd_una;
Ilpo Järvinen6859d492007-12-02 00:48:06 +02001500
1501 if (tp->highest_sack == NULL)
1502 return tp->snd_nxt;
1503
Ilpo Järvinena47e5a92007-11-15 19:41:46 -08001504 return TCP_SKB_CB(tp->highest_sack)->seq;
1505}
1506
Ilpo Järvinen6859d492007-12-02 00:48:06 +02001507static inline void tcp_advance_highest_sack(struct sock *sk, struct sk_buff *skb)
1508{
1509 tcp_sk(sk)->highest_sack = tcp_skb_is_last(sk, skb) ? NULL :
1510 tcp_write_queue_next(sk, skb);
1511}
1512
1513static inline struct sk_buff *tcp_highest_sack(struct sock *sk)
1514{
1515 return tcp_sk(sk)->highest_sack;
1516}
1517
1518static inline void tcp_highest_sack_reset(struct sock *sk)
1519{
1520 tcp_sk(sk)->highest_sack = tcp_write_queue_head(sk);
1521}
1522
1523/* Called when old skb is about to be deleted (to be combined with new skb) */
1524static inline void tcp_highest_sack_combine(struct sock *sk,
1525 struct sk_buff *old,
1526 struct sk_buff *new)
1527{
1528 if (tcp_sk(sk)->sacked_out && (old == tcp_sk(sk)->highest_sack))
1529 tcp_sk(sk)->highest_sack = new;
1530}
1531
Andreas Petlund5aa4b322010-02-18 02:45:45 +00001532/* Determines whether this is a thin stream (which may suffer from
1533 * increased latency). Used to trigger latency-reducing mechanisms.
1534 */
Eric Dumazeta2a385d2012-05-16 23:15:34 +00001535static inline bool tcp_stream_is_thin(struct tcp_sock *tp)
Andreas Petlund5aa4b322010-02-18 02:45:45 +00001536{
1537 return tp->packets_out < 4 && !tcp_in_initial_slowstart(tp);
1538}
1539
Linus Torvalds1da177e2005-04-16 15:20:36 -07001540/* /proc */
1541enum tcp_seq_states {
1542 TCP_SEQ_STATE_LISTENING,
1543 TCP_SEQ_STATE_OPENREQ,
1544 TCP_SEQ_STATE_ESTABLISHED,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001545};
1546
Arjan van de Ven73cb88e2011-10-30 06:46:30 +00001547int tcp_seq_open(struct inode *inode, struct file *file);
1548
Linus Torvalds1da177e2005-04-16 15:20:36 -07001549struct tcp_seq_afinfo {
Arjan van de Ven73cb88e2011-10-30 06:46:30 +00001550 char *name;
1551 sa_family_t family;
1552 const struct file_operations *seq_fops;
1553 struct seq_operations seq_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001554};
1555
1556struct tcp_iter_state {
Denis V. Luneva4146b12008-04-13 22:11:14 -07001557 struct seq_net_private p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001558 sa_family_t family;
1559 enum tcp_seq_states state;
1560 struct sock *syn_wait_sk;
Eric W. Biedermana7cb5a42012-05-24 01:10:10 -06001561 int bucket, offset, sbucket, num;
1562 kuid_t uid;
Tom Herberta8b690f2010-06-07 00:43:42 -07001563 loff_t last_pos;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001564};
1565
Joe Perches5c9f3022013-09-23 11:33:32 -07001566int tcp_proc_register(struct net *net, struct tcp_seq_afinfo *afinfo);
1567void tcp_proc_unregister(struct net *net, struct tcp_seq_afinfo *afinfo);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001568
Arnaldo Carvalho de Melo20380732005-08-16 02:18:02 -03001569extern struct request_sock_ops tcp_request_sock_ops;
Glenn Griffinc6aefaf2008-02-07 21:49:26 -08001570extern struct request_sock_ops tcp6_request_sock_ops;
Arnaldo Carvalho de Melo20380732005-08-16 02:18:02 -03001571
Joe Perches5c9f3022013-09-23 11:33:32 -07001572void tcp_v4_destroy_sock(struct sock *sk);
Arnaldo Carvalho de Melo20380732005-08-16 02:18:02 -03001573
Eric Dumazet28be6e02013-10-18 10:36:17 -07001574struct sk_buff *tcp_gso_segment(struct sk_buff *skb,
Joe Perches5c9f3022013-09-23 11:33:32 -07001575 netdev_features_t features);
1576struct sk_buff **tcp_gro_receive(struct sk_buff **head, struct sk_buff *skb);
1577int tcp_gro_complete(struct sk_buff *skb);
Daniel Borkmann28850dc2013-06-07 05:11:46 +00001578
Joe Perches5c9f3022013-09-23 11:33:32 -07001579void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, __be32 daddr);
Herbert Xuf4c50d92006-06-22 03:02:40 -07001580
Eric Dumazetc9bee3b72013-07-22 20:27:07 -07001581static inline u32 tcp_notsent_lowat(const struct tcp_sock *tp)
1582{
1583 return tp->notsent_lowat ?: sysctl_tcp_notsent_lowat;
1584}
1585
1586static inline bool tcp_stream_memory_free(const struct sock *sk)
1587{
1588 const struct tcp_sock *tp = tcp_sk(sk);
1589 u32 notsent_bytes = tp->write_seq - tp->snd_nxt;
1590
1591 return notsent_bytes < tcp_notsent_lowat(tp);
1592}
1593
Arnaldo Carvalho de Melo20380732005-08-16 02:18:02 -03001594#ifdef CONFIG_PROC_FS
Joe Perches5c9f3022013-09-23 11:33:32 -07001595int tcp4_proc_init(void);
1596void tcp4_proc_exit(void);
Arnaldo Carvalho de Melo20380732005-08-16 02:18:02 -03001597#endif
1598
Octavian Purdila5db92c92014-06-25 17:09:59 +03001599int tcp_rtx_synack(struct sock *sk, struct request_sock *req);
Octavian Purdila1fb6f152014-06-25 17:10:02 +03001600int tcp_conn_request(struct request_sock_ops *rsk_ops,
1601 const struct tcp_request_sock_ops *af_ops,
1602 struct sock *sk, struct sk_buff *skb);
Octavian Purdila5db92c92014-06-25 17:09:59 +03001603
YOSHIFUJI Hideakicfb6eeb2006-11-14 19:07:45 -08001604/* TCP af-specific functions */
1605struct tcp_sock_af_ops {
1606#ifdef CONFIG_TCP_MD5SIG
1607 struct tcp_md5sig_key *(*md5_lookup) (struct sock *sk,
1608 struct sock *addr_sk);
1609 int (*calc_md5_hash) (char *location,
1610 struct tcp_md5sig_key *md5,
Eric Dumazet318cf7a2011-10-24 02:46:04 -04001611 const struct sock *sk,
1612 const struct request_sock *req,
1613 const struct sk_buff *skb);
YOSHIFUJI Hideakicfb6eeb2006-11-14 19:07:45 -08001614 int (*md5_parse) (struct sock *sk,
1615 char __user *optval,
1616 int optlen);
1617#endif
1618};
1619
1620struct tcp_request_sock_ops {
Octavian Purdila2aec4a22014-06-25 17:10:00 +03001621 u16 mss_clamp;
YOSHIFUJI Hideakicfb6eeb2006-11-14 19:07:45 -08001622#ifdef CONFIG_TCP_MD5SIG
1623 struct tcp_md5sig_key *(*md5_lookup) (struct sock *sk,
1624 struct request_sock *req);
John Dykstrae3afe7b2009-07-16 05:04:51 +00001625 int (*calc_md5_hash) (char *location,
1626 struct tcp_md5sig_key *md5,
Eric Dumazet318cf7a2011-10-24 02:46:04 -04001627 const struct sock *sk,
1628 const struct request_sock *req,
1629 const struct sk_buff *skb);
YOSHIFUJI Hideakicfb6eeb2006-11-14 19:07:45 -08001630#endif
Octavian Purdila16bea702014-06-25 17:09:53 +03001631 void (*init_req)(struct request_sock *req, struct sock *sk,
1632 struct sk_buff *skb);
Octavian Purdilafb7b37a2014-06-25 17:09:54 +03001633#ifdef CONFIG_SYN_COOKIES
1634 __u32 (*cookie_init_seq)(struct sock *sk, const struct sk_buff *skb,
1635 __u16 *mss);
1636#endif
Octavian Purdilad94e0412014-06-25 17:09:55 +03001637 struct dst_entry *(*route_req)(struct sock *sk, struct flowi *fl,
1638 const struct request_sock *req,
1639 bool *strict);
Octavian Purdila936b8bd2014-06-25 17:09:57 +03001640 __u32 (*init_seq)(const struct sk_buff *skb);
Octavian Purdilad6274bd2014-06-25 17:09:58 +03001641 int (*send_synack)(struct sock *sk, struct dst_entry *dst,
1642 struct flowi *fl, struct request_sock *req,
1643 u16 queue_mapping, struct tcp_fastopen_cookie *foc);
Octavian Purdila695da142014-06-25 17:10:01 +03001644 void (*queue_hash_add)(struct sock *sk, struct request_sock *req,
1645 const unsigned long timeout);
YOSHIFUJI Hideakicfb6eeb2006-11-14 19:07:45 -08001646};
1647
Octavian Purdilafb7b37a2014-06-25 17:09:54 +03001648#ifdef CONFIG_SYN_COOKIES
1649static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops,
1650 struct sock *sk, struct sk_buff *skb,
1651 __u16 *mss)
1652{
1653 return ops->cookie_init_seq(sk, skb, mss);
1654}
1655#else
1656static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops,
1657 struct sock *sk, struct sk_buff *skb,
1658 __u16 *mss)
1659{
1660 return 0;
1661}
1662#endif
1663
Joe Perches5c9f3022013-09-23 11:33:32 -07001664int tcpv4_offload_init(void);
Daniel Borkmann28850dc2013-06-07 05:11:46 +00001665
Joe Perches5c9f3022013-09-23 11:33:32 -07001666void tcp_v4_init(void);
1667void tcp_init(void);
Arnaldo Carvalho de Melo20380732005-08-16 02:18:02 -03001668
Cong Wange25f8662014-10-15 14:33:21 -07001669/*
1670 * Save and compile IPv4 options, return a pointer to it
1671 */
1672static inline struct ip_options_rcu *tcp_v4_save_options(struct sk_buff *skb)
1673{
1674 const struct ip_options *opt = &TCP_SKB_CB(skb)->header.h4.opt;
1675 struct ip_options_rcu *dopt = NULL;
1676
1677 if (opt && opt->optlen) {
1678 int opt_size = sizeof(*dopt) + opt->optlen;
1679
1680 dopt = kmalloc(opt_size, GFP_ATOMIC);
1681 if (dopt && __ip_options_echo(&dopt->opt, skb, opt)) {
1682 kfree(dopt);
1683 dopt = NULL;
1684 }
1685 }
1686 return dopt;
1687}
1688
Linus Torvalds1da177e2005-04-16 15:20:36 -07001689#endif /* _TCP_H */