blob: 79c39dc9b01c6f0d147a952628a42a81534704b9 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * Implementation of the Transmission Control Protocol(TCP).
7 *
Jesper Juhl02c30a82005-05-05 16:16:16 -07008 * Authors: Ross Biro
Linus Torvalds1da177e2005-04-16 15:20:36 -07009 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Mark Evans, <evansmp@uhura.aston.ac.uk>
11 * Corey Minyard <wf-rch!minyard@relay.EU.net>
12 * Florian La Roche, <flla@stud.uni-sb.de>
13 * Charles Hedrick, <hedrick@klinzhai.rutgers.edu>
14 * Linus Torvalds, <torvalds@cs.helsinki.fi>
15 * Alan Cox, <gw4pts@gw4pts.ampr.org>
16 * Matthew Dillon, <dillon@apollo.west.oic.com>
17 * Arnt Gulbrandsen, <agulbra@nvg.unit.no>
18 * Jorge Cwik, <jorge@laser.satlink.net>
19 */
20
21/*
22 * Changes: Pedro Roque : Retransmit queue handled by TCP.
23 * : Fragmentation on mtu decrease
24 * : Segment collapse on retransmit
25 * : AF independence
26 *
27 * Linus Torvalds : send_delayed_ack
28 * David S. Miller : Charge memory using the right skb
29 * during syn/ack processing.
30 * David S. Miller : Output engine completely rewritten.
31 * Andrea Arcangeli: SYNACK carry ts_recent in tsecr.
32 * Cacophonix Gaul : draft-minshall-nagle-01
33 * J Hadi Salim : ECN support
34 *
35 */
36
37#include <net/tcp.h>
38
39#include <linux/compiler.h>
40#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070041
42/* People can turn this off for buggy TCP's found in printers etc. */
Brian Haleyab32ea52006-09-22 14:15:41 -070043int sysctl_tcp_retrans_collapse __read_mostly = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -070044
Jianjun Kong09cb1052008-11-03 00:27:11 -080045/* People can turn this on to work with those rare, broken TCPs that
Rick Jones15d99e02006-03-20 22:40:29 -080046 * interpret the window field as a signed quantity.
47 */
Brian Haleyab32ea52006-09-22 14:15:41 -070048int sysctl_tcp_workaround_signed_windows __read_mostly = 0;
Rick Jones15d99e02006-03-20 22:40:29 -080049
Linus Torvalds1da177e2005-04-16 15:20:36 -070050/* This limits the percentage of the congestion window which we
51 * will allow a single TSO frame to consume. Building TSO frames
52 * which are too large can cause TCP streams to be bursty.
53 */
Brian Haleyab32ea52006-09-22 14:15:41 -070054int sysctl_tcp_tso_win_divisor __read_mostly = 3;
Linus Torvalds1da177e2005-04-16 15:20:36 -070055
Brian Haleyab32ea52006-09-22 14:15:41 -070056int sysctl_tcp_mtu_probing __read_mostly = 0;
57int sysctl_tcp_base_mss __read_mostly = 512;
John Heffner5d424d52006-03-20 17:53:41 -080058
David S. Miller35089bb2006-06-13 22:33:04 -070059/* By default, RFC2861 behavior. */
Brian Haleyab32ea52006-09-22 14:15:41 -070060int sysctl_tcp_slow_start_after_idle __read_mostly = 1;
David S. Miller35089bb2006-06-13 22:33:04 -070061
Ilpo Järvinen66f5fe62007-12-31 04:43:57 -080062static void tcp_event_new_data_sent(struct sock *sk, struct sk_buff *skb)
Ilpo Järvinen6ff03ac2007-08-24 22:44:06 -070063{
64 struct tcp_sock *tp = tcp_sk(sk);
Ilpo Järvinen66f5fe62007-12-31 04:43:57 -080065 unsigned int prior_packets = tp->packets_out;
Ilpo Järvinen9e412ba2007-04-20 22:18:02 -070066
David S. Millerfe067e82007-03-07 12:12:44 -080067 tcp_advance_send_head(sk, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -070068 tp->snd_nxt = TCP_SKB_CB(skb)->end_seq;
Ilpo Järvinen85124302007-11-26 20:17:38 +080069
70 /* Don't override Nagle indefinately with F-RTO */
71 if (tp->frto_counter == 2)
72 tp->frto_counter = 3;
Ilpo Järvinen66f5fe62007-12-31 04:43:57 -080073
74 tp->packets_out += tcp_skb_pcount(skb);
75 if (!prior_packets)
76 inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
77 inet_csk(sk)->icsk_rto, TCP_RTO_MAX);
Linus Torvalds1da177e2005-04-16 15:20:36 -070078}
79
80/* SND.NXT, if window was not shrunk.
81 * If window has been shrunk, what should we make? It is not clear at all.
82 * Using SND.UNA we will fail to open window, SND.NXT is out of window. :-(
83 * Anything in between SND.UNA...SND.UNA+SND.WND also can be already
84 * invalid. OK, let's make this for now:
85 */
Ilpo Järvinen9e412ba2007-04-20 22:18:02 -070086static inline __u32 tcp_acceptable_seq(struct sock *sk)
Linus Torvalds1da177e2005-04-16 15:20:36 -070087{
Ilpo Järvinen9e412ba2007-04-20 22:18:02 -070088 struct tcp_sock *tp = tcp_sk(sk);
89
Ilpo Järvinen90840de2007-12-31 04:48:41 -080090 if (!before(tcp_wnd_end(tp), tp->snd_nxt))
Linus Torvalds1da177e2005-04-16 15:20:36 -070091 return tp->snd_nxt;
92 else
Ilpo Järvinen90840de2007-12-31 04:48:41 -080093 return tcp_wnd_end(tp);
Linus Torvalds1da177e2005-04-16 15:20:36 -070094}
95
96/* Calculate mss to advertise in SYN segment.
97 * RFC1122, RFC1063, draft-ietf-tcpimpl-pmtud-01 state that:
98 *
99 * 1. It is independent of path mtu.
100 * 2. Ideally, it is maximal possible segment size i.e. 65535-40.
101 * 3. For IPv4 it is reasonable to calculate it from maximal MTU of
102 * attached devices, because some buggy hosts are confused by
103 * large MSS.
104 * 4. We do not make 3, we advertise MSS, calculated from first
105 * hop device mtu, but allow to raise it to ip_rt_min_advmss.
106 * This may be overridden via information stored in routing table.
107 * 5. Value 65535 for MSS is valid in IPv6 and means "as large as possible,
108 * probably even Jumbo".
109 */
110static __u16 tcp_advertise_mss(struct sock *sk)
111{
112 struct tcp_sock *tp = tcp_sk(sk);
113 struct dst_entry *dst = __sk_dst_get(sk);
114 int mss = tp->advmss;
115
116 if (dst && dst_metric(dst, RTAX_ADVMSS) < mss) {
117 mss = dst_metric(dst, RTAX_ADVMSS);
118 tp->advmss = mss;
119 }
120
121 return (__u16)mss;
122}
123
124/* RFC2861. Reset CWND after idle period longer RTO to "restart window".
125 * This is the first part of cwnd validation mechanism. */
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -0700126static void tcp_cwnd_restart(struct sock *sk, struct dst_entry *dst)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700127{
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -0700128 struct tcp_sock *tp = tcp_sk(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700129 s32 delta = tcp_time_stamp - tp->lsndtime;
130 u32 restart_cwnd = tcp_init_cwnd(tp, dst);
131 u32 cwnd = tp->snd_cwnd;
132
Arnaldo Carvalho de Melo6687e982005-08-10 04:03:31 -0300133 tcp_ca_event(sk, CA_EVENT_CWND_RESTART);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700134
Arnaldo Carvalho de Melo6687e982005-08-10 04:03:31 -0300135 tp->snd_ssthresh = tcp_current_ssthresh(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700136 restart_cwnd = min(restart_cwnd, cwnd);
137
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -0700138 while ((delta -= inet_csk(sk)->icsk_rto) > 0 && cwnd > restart_cwnd)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700139 cwnd >>= 1;
140 tp->snd_cwnd = max(cwnd, restart_cwnd);
141 tp->snd_cwnd_stamp = tcp_time_stamp;
142 tp->snd_cwnd_used = 0;
143}
144
Stephen Hemminger40efc6f2006-01-03 16:03:49 -0800145static void tcp_event_data_sent(struct tcp_sock *tp,
146 struct sk_buff *skb, struct sock *sk)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700147{
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -0700148 struct inet_connection_sock *icsk = inet_csk(sk);
149 const u32 now = tcp_time_stamp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700150
David S. Miller35089bb2006-06-13 22:33:04 -0700151 if (sysctl_tcp_slow_start_after_idle &&
152 (!tp->packets_out && (s32)(now - tp->lsndtime) > icsk->icsk_rto))
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -0700153 tcp_cwnd_restart(sk, __sk_dst_get(sk));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700154
155 tp->lsndtime = now;
156
157 /* If it is a reply for ato after last received
158 * packet, enter pingpong mode.
159 */
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -0700160 if ((u32)(now - icsk->icsk_ack.lrcvtime) < icsk->icsk_ack.ato)
161 icsk->icsk_ack.pingpong = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700162}
163
Stephen Hemminger40efc6f2006-01-03 16:03:49 -0800164static inline void tcp_event_ack_sent(struct sock *sk, unsigned int pkts)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700165{
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -0700166 tcp_dec_quickack_mode(sk, pkts);
167 inet_csk_clear_xmit_timer(sk, ICSK_TIME_DACK);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700168}
169
170/* Determine a window scaling and initial window to offer.
171 * Based on the assumption that the given amount of space
172 * will be offered. Store the results in the tp structure.
173 * NOTE: for smooth operation initial space offering should
174 * be a multiple of mss if possible. We assume here that mss >= 1.
175 * This MUST be enforced by all callers.
176 */
177void tcp_select_initial_window(int __space, __u32 mss,
178 __u32 *rcv_wnd, __u32 *window_clamp,
179 int wscale_ok, __u8 *rcv_wscale)
180{
181 unsigned int space = (__space < 0 ? 0 : __space);
182
183 /* If no clamp set the clamp to the max possible scaled window */
184 if (*window_clamp == 0)
185 (*window_clamp) = (65535 << 14);
186 space = min(*window_clamp, space);
187
188 /* Quantize space offering to a multiple of mss if possible. */
189 if (space > mss)
190 space = (space / mss) * mss;
191
192 /* NOTE: offering an initial window larger than 32767
Rick Jones15d99e02006-03-20 22:40:29 -0800193 * will break some buggy TCP stacks. If the admin tells us
194 * it is likely we could be speaking with such a buggy stack
195 * we will truncate our initial window offering to 32K-1
196 * unless the remote has sent us a window scaling option,
197 * which we interpret as a sign the remote TCP is not
198 * misinterpreting the window field as a signed quantity.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700199 */
Rick Jones15d99e02006-03-20 22:40:29 -0800200 if (sysctl_tcp_workaround_signed_windows)
201 (*rcv_wnd) = min(space, MAX_TCP_WINDOW);
202 else
203 (*rcv_wnd) = space;
204
Linus Torvalds1da177e2005-04-16 15:20:36 -0700205 (*rcv_wscale) = 0;
206 if (wscale_ok) {
207 /* Set window scaling on max possible window
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900208 * See RFC1323 for an explanation of the limit to 14
Linus Torvalds1da177e2005-04-16 15:20:36 -0700209 */
210 space = max_t(u32, sysctl_tcp_rmem[2], sysctl_rmem_max);
Stephen Hemminger316c1592006-08-22 00:06:11 -0700211 space = min_t(u32, space, *window_clamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700212 while (space > 65535 && (*rcv_wscale) < 14) {
213 space >>= 1;
214 (*rcv_wscale)++;
215 }
216 }
217
218 /* Set initial window to value enough for senders,
David S. Miller6b251852005-09-28 16:31:48 -0700219 * following RFC2414. Senders, not following this RFC,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700220 * will be satisfied with 2.
221 */
Ilpo Järvinen056834d2007-12-31 14:57:14 -0800222 if (mss > (1 << *rcv_wscale)) {
David S. Miller01ff3672005-09-29 17:07:20 -0700223 int init_cwnd = 4;
Ilpo Järvinen056834d2007-12-31 14:57:14 -0800224 if (mss > 1460 * 3)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700225 init_cwnd = 2;
David S. Miller01ff3672005-09-29 17:07:20 -0700226 else if (mss > 1460)
227 init_cwnd = 3;
Ilpo Järvinen056834d2007-12-31 14:57:14 -0800228 if (*rcv_wnd > init_cwnd * mss)
229 *rcv_wnd = init_cwnd * mss;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700230 }
231
232 /* Set the clamp no higher than max representable value */
233 (*window_clamp) = min(65535U << (*rcv_wscale), *window_clamp);
234}
235
236/* Chose a new window to advertise, update state in tcp_sock for the
237 * socket, and return result with RFC1323 scaling applied. The return
238 * value can be stuffed directly into th->window for an outgoing
239 * frame.
240 */
Stephen Hemminger40efc6f2006-01-03 16:03:49 -0800241static u16 tcp_select_window(struct sock *sk)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700242{
243 struct tcp_sock *tp = tcp_sk(sk);
244 u32 cur_win = tcp_receive_window(tp);
245 u32 new_win = __tcp_select_window(sk);
246
247 /* Never shrink the offered window */
Stephen Hemminger2de979b2007-03-08 20:45:19 -0800248 if (new_win < cur_win) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700249 /* Danger Will Robinson!
250 * Don't update rcv_wup/rcv_wnd here or else
251 * we will not be able to advertise a zero
252 * window in time. --DaveM
253 *
254 * Relax Will Robinson.
255 */
Patrick McHardy607bfbf2008-03-20 16:11:27 -0700256 new_win = ALIGN(cur_win, 1 << tp->rx_opt.rcv_wscale);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700257 }
258 tp->rcv_wnd = new_win;
259 tp->rcv_wup = tp->rcv_nxt;
260
261 /* Make sure we do not exceed the maximum possible
262 * scaled window.
263 */
Rick Jones15d99e02006-03-20 22:40:29 -0800264 if (!tp->rx_opt.rcv_wscale && sysctl_tcp_workaround_signed_windows)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700265 new_win = min(new_win, MAX_TCP_WINDOW);
266 else
267 new_win = min(new_win, (65535U << tp->rx_opt.rcv_wscale));
268
269 /* RFC1323 scaling applied */
270 new_win >>= tp->rx_opt.rcv_wscale;
271
272 /* If we advertise zero window, disable fast path. */
273 if (new_win == 0)
274 tp->pred_flags = 0;
275
276 return new_win;
277}
278
Ilpo Järvinen056834d2007-12-31 14:57:14 -0800279static inline void TCP_ECN_send_synack(struct tcp_sock *tp, struct sk_buff *skb)
Ilpo Järvinenbdf1ee52007-05-27 02:04:16 -0700280{
281 TCP_SKB_CB(skb)->flags &= ~TCPCB_FLAG_CWR;
Ilpo Järvinen056834d2007-12-31 14:57:14 -0800282 if (!(tp->ecn_flags & TCP_ECN_OK))
Ilpo Järvinenbdf1ee52007-05-27 02:04:16 -0700283 TCP_SKB_CB(skb)->flags &= ~TCPCB_FLAG_ECE;
284}
285
286static inline void TCP_ECN_send_syn(struct sock *sk, struct sk_buff *skb)
287{
288 struct tcp_sock *tp = tcp_sk(sk);
289
290 tp->ecn_flags = 0;
Ilpo Järvinen255cac92009-05-04 11:07:36 -0700291 if (sysctl_tcp_ecn == 1) {
Ilpo Järvinen056834d2007-12-31 14:57:14 -0800292 TCP_SKB_CB(skb)->flags |= TCPCB_FLAG_ECE | TCPCB_FLAG_CWR;
Ilpo Järvinenbdf1ee52007-05-27 02:04:16 -0700293 tp->ecn_flags = TCP_ECN_OK;
294 }
295}
296
297static __inline__ void
298TCP_ECN_make_synack(struct request_sock *req, struct tcphdr *th)
299{
300 if (inet_rsk(req)->ecn_ok)
301 th->ece = 1;
302}
303
304static inline void TCP_ECN_send(struct sock *sk, struct sk_buff *skb,
305 int tcp_header_len)
306{
307 struct tcp_sock *tp = tcp_sk(sk);
308
309 if (tp->ecn_flags & TCP_ECN_OK) {
310 /* Not-retransmitted data segment: set ECT and inject CWR. */
311 if (skb->len != tcp_header_len &&
312 !before(TCP_SKB_CB(skb)->seq, tp->snd_nxt)) {
313 INET_ECN_xmit(sk);
Ilpo Järvinen056834d2007-12-31 14:57:14 -0800314 if (tp->ecn_flags & TCP_ECN_QUEUE_CWR) {
Ilpo Järvinenbdf1ee52007-05-27 02:04:16 -0700315 tp->ecn_flags &= ~TCP_ECN_QUEUE_CWR;
316 tcp_hdr(skb)->cwr = 1;
317 skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;
318 }
319 } else {
320 /* ACK or retransmitted segment: clear ECT|CE */
321 INET_ECN_dontxmit(sk);
322 }
323 if (tp->ecn_flags & TCP_ECN_DEMAND_CWR)
324 tcp_hdr(skb)->ece = 1;
325 }
326}
327
Ilpo Järvinene870a8e2008-01-03 20:39:01 -0800328/* Constructs common control bits of non-data skb. If SYN/FIN is present,
329 * auto increment end seqno.
330 */
331static void tcp_init_nondata_skb(struct sk_buff *skb, u32 seq, u8 flags)
332{
333 skb->csum = 0;
334
335 TCP_SKB_CB(skb)->flags = flags;
336 TCP_SKB_CB(skb)->sacked = 0;
337
338 skb_shinfo(skb)->gso_segs = 1;
339 skb_shinfo(skb)->gso_size = 0;
340 skb_shinfo(skb)->gso_type = 0;
341
342 TCP_SKB_CB(skb)->seq = seq;
343 if (flags & (TCPCB_FLAG_SYN | TCPCB_FLAG_FIN))
344 seq++;
345 TCP_SKB_CB(skb)->end_seq = seq;
346}
347
Ilpo Järvinen33f5f572008-10-07 14:43:06 -0700348static inline int tcp_urg_mode(const struct tcp_sock *tp)
349{
350 return tp->snd_una != tp->snd_up;
351}
352
Adam Langley33ad7982008-07-19 00:04:31 -0700353#define OPTION_SACK_ADVERTISE (1 << 0)
354#define OPTION_TS (1 << 1)
355#define OPTION_MD5 (1 << 2)
356
357struct tcp_out_options {
358 u8 options; /* bit field of OPTION_* */
359 u8 ws; /* window scale, 0 to disable */
360 u8 num_sack_blocks; /* number of SACK blocks to include */
361 u16 mss; /* 0 to disable */
362 __u32 tsval, tsecr; /* need to include OPTION_TS */
363};
364
Ilpo Järvinenfd6149d2008-10-23 14:06:35 -0700365/* Beware: Something in the Internet is very sensitive to the ordering of
366 * TCP options, we learned this through the hard way, so be careful here.
367 * Luckily we can at least blame others for their non-compliance but from
368 * inter-operatibility perspective it seems that we're somewhat stuck with
369 * the ordering which we have been using if we want to keep working with
370 * those broken things (not that it currently hurts anybody as there isn't
371 * particular reason why the ordering would need to be changed).
372 *
373 * At least SACK_PERM as the first option is known to lead to a disaster
374 * (but it may well be that other scenarios fail similarly).
375 */
Adam Langley33ad7982008-07-19 00:04:31 -0700376static void tcp_options_write(__be32 *ptr, struct tcp_sock *tp,
377 const struct tcp_out_options *opts,
378 __u8 **md5_hash) {
379 if (unlikely(OPTION_MD5 & opts->options)) {
YOSHIFUJI Hideaki496c98d2006-10-10 19:41:21 -0700380 *ptr++ = htonl((TCPOPT_NOP << 24) |
381 (TCPOPT_NOP << 16) |
Adam Langley33ad7982008-07-19 00:04:31 -0700382 (TCPOPT_MD5SIG << 8) |
383 TCPOLEN_MD5SIG);
384 *md5_hash = (__u8 *)ptr;
385 ptr += 4;
386 } else {
387 *md5_hash = NULL;
Stephen Hemminger40efc6f2006-01-03 16:03:49 -0800388 }
Adam Langley33ad7982008-07-19 00:04:31 -0700389
Ilpo Järvinenfd6149d2008-10-23 14:06:35 -0700390 if (unlikely(opts->mss)) {
391 *ptr++ = htonl((TCPOPT_MSS << 24) |
392 (TCPOLEN_MSS << 16) |
393 opts->mss);
394 }
395
Adam Langley33ad7982008-07-19 00:04:31 -0700396 if (likely(OPTION_TS & opts->options)) {
397 if (unlikely(OPTION_SACK_ADVERTISE & opts->options)) {
398 *ptr++ = htonl((TCPOPT_SACK_PERM << 24) |
399 (TCPOLEN_SACK_PERM << 16) |
400 (TCPOPT_TIMESTAMP << 8) |
401 TCPOLEN_TIMESTAMP);
402 } else {
403 *ptr++ = htonl((TCPOPT_NOP << 24) |
404 (TCPOPT_NOP << 16) |
405 (TCPOPT_TIMESTAMP << 8) |
406 TCPOLEN_TIMESTAMP);
407 }
408 *ptr++ = htonl(opts->tsval);
409 *ptr++ = htonl(opts->tsecr);
410 }
411
Adam Langley33ad7982008-07-19 00:04:31 -0700412 if (unlikely(OPTION_SACK_ADVERTISE & opts->options &&
413 !(OPTION_TS & opts->options))) {
414 *ptr++ = htonl((TCPOPT_NOP << 24) |
415 (TCPOPT_NOP << 16) |
416 (TCPOPT_SACK_PERM << 8) |
417 TCPOLEN_SACK_PERM);
418 }
419
420 if (unlikely(opts->ws)) {
421 *ptr++ = htonl((TCPOPT_NOP << 24) |
422 (TCPOPT_WINDOW << 16) |
423 (TCPOLEN_WINDOW << 8) |
424 opts->ws);
425 }
426
427 if (unlikely(opts->num_sack_blocks)) {
428 struct tcp_sack_block *sp = tp->rx_opt.dsack ?
429 tp->duplicate_sack : tp->selective_acks;
Stephen Hemminger40efc6f2006-01-03 16:03:49 -0800430 int this_sack;
431
432 *ptr++ = htonl((TCPOPT_NOP << 24) |
433 (TCPOPT_NOP << 16) |
434 (TCPOPT_SACK << 8) |
Adam Langley33ad7982008-07-19 00:04:31 -0700435 (TCPOLEN_SACK_BASE + (opts->num_sack_blocks *
Stephen Hemminger40efc6f2006-01-03 16:03:49 -0800436 TCPOLEN_SACK_PERBLOCK)));
Stephen Hemminger2de979b2007-03-08 20:45:19 -0800437
Adam Langley33ad7982008-07-19 00:04:31 -0700438 for (this_sack = 0; this_sack < opts->num_sack_blocks;
439 ++this_sack) {
Stephen Hemminger40efc6f2006-01-03 16:03:49 -0800440 *ptr++ = htonl(sp[this_sack].start_seq);
441 *ptr++ = htonl(sp[this_sack].end_seq);
442 }
Stephen Hemminger2de979b2007-03-08 20:45:19 -0800443
Ilpo Järvinen5861f8e2009-03-14 14:23:01 +0000444 tp->rx_opt.dsack = 0;
Stephen Hemminger40efc6f2006-01-03 16:03:49 -0800445 }
446}
447
Adam Langley33ad7982008-07-19 00:04:31 -0700448static unsigned tcp_syn_options(struct sock *sk, struct sk_buff *skb,
449 struct tcp_out_options *opts,
450 struct tcp_md5sig_key **md5) {
451 struct tcp_sock *tp = tcp_sk(sk);
452 unsigned size = 0;
453
YOSHIFUJI Hideakicfb6eeb2006-11-14 19:07:45 -0800454#ifdef CONFIG_TCP_MD5SIG
Adam Langley33ad7982008-07-19 00:04:31 -0700455 *md5 = tp->af_specific->md5_lookup(sk, sk);
456 if (*md5) {
457 opts->options |= OPTION_MD5;
458 size += TCPOLEN_MD5SIG_ALIGNED;
YOSHIFUJI Hideakicfb6eeb2006-11-14 19:07:45 -0800459 }
Adam Langley33ad7982008-07-19 00:04:31 -0700460#else
461 *md5 = NULL;
YOSHIFUJI Hideakicfb6eeb2006-11-14 19:07:45 -0800462#endif
Adam Langley33ad7982008-07-19 00:04:31 -0700463
464 /* We always get an MSS option. The option bytes which will be seen in
465 * normal data packets should timestamps be used, must be in the MSS
466 * advertised. But we subtract them from tp->mss_cache so that
467 * calculations in tcp_sendmsg are simpler etc. So account for this
468 * fact here if necessary. If we don't do this correctly, as a
469 * receiver we won't recognize data packets as being full sized when we
470 * should, and thus we won't abide by the delayed ACK rules correctly.
471 * SACKs don't matter, we never delay an ACK when we have any of those
472 * going out. */
473 opts->mss = tcp_advertise_mss(sk);
474 size += TCPOLEN_MSS_ALIGNED;
475
476 if (likely(sysctl_tcp_timestamps && *md5 == NULL)) {
477 opts->options |= OPTION_TS;
478 opts->tsval = TCP_SKB_CB(skb)->when;
479 opts->tsecr = tp->rx_opt.ts_recent;
480 size += TCPOLEN_TSTAMP_ALIGNED;
481 }
482 if (likely(sysctl_tcp_window_scaling)) {
483 opts->ws = tp->rx_opt.rcv_wscale;
Jianjun Kong09cb1052008-11-03 00:27:11 -0800484 if (likely(opts->ws))
Philip Love7982d5e2008-08-27 02:33:50 -0700485 size += TCPOLEN_WSCALE_ALIGNED;
Adam Langley33ad7982008-07-19 00:04:31 -0700486 }
487 if (likely(sysctl_tcp_sack)) {
488 opts->options |= OPTION_SACK_ADVERTISE;
David S. Millerb32d1312008-07-21 18:45:34 -0700489 if (unlikely(!(OPTION_TS & opts->options)))
Adam Langley33ad7982008-07-19 00:04:31 -0700490 size += TCPOLEN_SACKPERM_ALIGNED;
491 }
492
493 return size;
494}
495
496static unsigned tcp_synack_options(struct sock *sk,
497 struct request_sock *req,
498 unsigned mss, struct sk_buff *skb,
499 struct tcp_out_options *opts,
500 struct tcp_md5sig_key **md5) {
501 unsigned size = 0;
502 struct inet_request_sock *ireq = inet_rsk(req);
503 char doing_ts;
504
505#ifdef CONFIG_TCP_MD5SIG
506 *md5 = tcp_rsk(req)->af_specific->md5_lookup(sk, req);
507 if (*md5) {
508 opts->options |= OPTION_MD5;
509 size += TCPOLEN_MD5SIG_ALIGNED;
510 }
511#else
512 *md5 = NULL;
513#endif
514
515 /* we can't fit any SACK blocks in a packet with MD5 + TS
516 options. There was discussion about disabling SACK rather than TS in
517 order to fit in better with old, buggy kernels, but that was deemed
518 to be unnecessary. */
519 doing_ts = ireq->tstamp_ok && !(*md5 && ireq->sack_ok);
520
521 opts->mss = mss;
522 size += TCPOLEN_MSS_ALIGNED;
523
524 if (likely(ireq->wscale_ok)) {
525 opts->ws = ireq->rcv_wscale;
Jianjun Kong09cb1052008-11-03 00:27:11 -0800526 if (likely(opts->ws))
Philip Love7982d5e2008-08-27 02:33:50 -0700527 size += TCPOLEN_WSCALE_ALIGNED;
Adam Langley33ad7982008-07-19 00:04:31 -0700528 }
529 if (likely(doing_ts)) {
530 opts->options |= OPTION_TS;
531 opts->tsval = TCP_SKB_CB(skb)->when;
532 opts->tsecr = req->ts_recent;
533 size += TCPOLEN_TSTAMP_ALIGNED;
534 }
535 if (likely(ireq->sack_ok)) {
536 opts->options |= OPTION_SACK_ADVERTISE;
537 if (unlikely(!doing_ts))
538 size += TCPOLEN_SACKPERM_ALIGNED;
539 }
540
541 return size;
542}
543
544static unsigned tcp_established_options(struct sock *sk, struct sk_buff *skb,
545 struct tcp_out_options *opts,
546 struct tcp_md5sig_key **md5) {
547 struct tcp_skb_cb *tcb = skb ? TCP_SKB_CB(skb) : NULL;
548 struct tcp_sock *tp = tcp_sk(sk);
549 unsigned size = 0;
Ilpo Järvinencabeccb2009-02-28 04:44:38 +0000550 unsigned int eff_sacks;
Adam Langley33ad7982008-07-19 00:04:31 -0700551
552#ifdef CONFIG_TCP_MD5SIG
553 *md5 = tp->af_specific->md5_lookup(sk, sk);
554 if (unlikely(*md5)) {
555 opts->options |= OPTION_MD5;
556 size += TCPOLEN_MD5SIG_ALIGNED;
557 }
558#else
559 *md5 = NULL;
560#endif
561
562 if (likely(tp->rx_opt.tstamp_ok)) {
563 opts->options |= OPTION_TS;
564 opts->tsval = tcb ? tcb->when : 0;
565 opts->tsecr = tp->rx_opt.ts_recent;
566 size += TCPOLEN_TSTAMP_ALIGNED;
567 }
568
Ilpo Järvinencabeccb2009-02-28 04:44:38 +0000569 eff_sacks = tp->rx_opt.num_sacks + tp->rx_opt.dsack;
570 if (unlikely(eff_sacks)) {
Adam Langley33ad7982008-07-19 00:04:31 -0700571 const unsigned remaining = MAX_TCP_OPTION_SPACE - size;
572 opts->num_sack_blocks =
Ilpo Järvinencabeccb2009-02-28 04:44:38 +0000573 min_t(unsigned, eff_sacks,
Adam Langley33ad7982008-07-19 00:04:31 -0700574 (remaining - TCPOLEN_SACK_BASE_ALIGNED) /
575 TCPOLEN_SACK_PERBLOCK);
576 size += TCPOLEN_SACK_BASE_ALIGNED +
577 opts->num_sack_blocks * TCPOLEN_SACK_PERBLOCK;
578 }
579
580 return size;
Stephen Hemminger40efc6f2006-01-03 16:03:49 -0800581}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700582
583/* This routine actually transmits TCP packets queued in by
584 * tcp_do_sendmsg(). This is used by both the initial
585 * transmission and possible later retransmissions.
586 * All SKB's seen here are completely headerless. It is our
587 * job to build the TCP header, and pass the packet down to
588 * IP so it can do the same plus pass the packet off to the
589 * device.
590 *
591 * We are working here with either a clone of the original
592 * SKB, or a fresh unique copy made by the retransmit engine.
593 */
Ilpo Järvinen056834d2007-12-31 14:57:14 -0800594static int tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, int clone_it,
595 gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700596{
David S. Millerdfb4b9d2005-12-06 16:24:52 -0800597 const struct inet_connection_sock *icsk = inet_csk(sk);
598 struct inet_sock *inet;
599 struct tcp_sock *tp;
600 struct tcp_skb_cb *tcb;
Adam Langley33ad7982008-07-19 00:04:31 -0700601 struct tcp_out_options opts;
602 unsigned tcp_options_size, tcp_header_size;
YOSHIFUJI Hideakicfb6eeb2006-11-14 19:07:45 -0800603 struct tcp_md5sig_key *md5;
604 __u8 *md5_hash_location;
David S. Millerdfb4b9d2005-12-06 16:24:52 -0800605 struct tcphdr *th;
David S. Millerdfb4b9d2005-12-06 16:24:52 -0800606 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700607
David S. Millerdfb4b9d2005-12-06 16:24:52 -0800608 BUG_ON(!skb || !tcp_skb_pcount(skb));
609
610 /* If congestion control is doing timestamping, we must
611 * take such a timestamp before we potentially clone/copy.
612 */
Stephen Hemminger164891a2007-04-23 22:26:16 -0700613 if (icsk->icsk_ca_ops->flags & TCP_CONG_RTT_STAMP)
David S. Millerdfb4b9d2005-12-06 16:24:52 -0800614 __net_timestamp(skb);
615
616 if (likely(clone_it)) {
617 if (unlikely(skb_cloned(skb)))
618 skb = pskb_copy(skb, gfp_mask);
619 else
620 skb = skb_clone(skb, gfp_mask);
621 if (unlikely(!skb))
622 return -ENOBUFS;
623 }
624
625 inet = inet_sk(sk);
626 tp = tcp_sk(sk);
627 tcb = TCP_SKB_CB(skb);
Adam Langley33ad7982008-07-19 00:04:31 -0700628 memset(&opts, 0, sizeof(opts));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700629
Adam Langley33ad7982008-07-19 00:04:31 -0700630 if (unlikely(tcb->flags & TCPCB_FLAG_SYN))
631 tcp_options_size = tcp_syn_options(sk, skb, &opts, &md5);
632 else
633 tcp_options_size = tcp_established_options(sk, skb, &opts,
634 &md5);
635 tcp_header_size = tcp_options_size + sizeof(struct tcphdr);
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900636
David S. Millerdfb4b9d2005-12-06 16:24:52 -0800637 if (tcp_packets_in_flight(tp) == 0)
638 tcp_ca_event(sk, CA_EVENT_TX_START);
639
Arnaldo Carvalho de Meloaa8223c2007-04-10 21:04:22 -0700640 skb_push(skb, tcp_header_size);
641 skb_reset_transport_header(skb);
David S. Millere89862f2007-01-26 01:04:55 -0800642 skb_set_owner_w(skb, sk);
David S. Millerdfb4b9d2005-12-06 16:24:52 -0800643
644 /* Build TCP header and checksum it. */
Arnaldo Carvalho de Meloaa8223c2007-04-10 21:04:22 -0700645 th = tcp_hdr(skb);
David S. Millerdfb4b9d2005-12-06 16:24:52 -0800646 th->source = inet->sport;
647 th->dest = inet->dport;
648 th->seq = htonl(tcb->seq);
649 th->ack_seq = htonl(tp->rcv_nxt);
Al Virodf7a3b02006-09-27 18:38:52 -0700650 *(((__be16 *)th) + 6) = htons(((tcp_header_size >> 2) << 12) |
David S. Millerdfb4b9d2005-12-06 16:24:52 -0800651 tcb->flags);
652
653 if (unlikely(tcb->flags & TCPCB_FLAG_SYN)) {
654 /* RFC1323: The window in SYN & SYN/ACK segments
655 * is never scaled.
656 */
Ilpo Järvinen600ff0c2007-02-13 12:42:11 -0800657 th->window = htons(min(tp->rcv_wnd, 65535U));
David S. Millerdfb4b9d2005-12-06 16:24:52 -0800658 } else {
659 th->window = htons(tcp_select_window(sk));
660 }
661 th->check = 0;
662 th->urg_ptr = 0;
663
Ilpo Järvinen33f5f572008-10-07 14:43:06 -0700664 /* The urg_mode check is necessary during a below snd_una win probe */
Herbert Xu76913672009-02-21 23:52:29 -0800665 if (unlikely(tcp_urg_mode(tp) && before(tcb->seq, tp->snd_up))) {
666 if (before(tp->snd_up, tcb->seq + 0x10000)) {
667 th->urg_ptr = htons(tp->snd_up - tcb->seq);
668 th->urg = 1;
669 } else if (after(tcb->seq + 0xFFFF, tp->snd_nxt)) {
670 th->urg_ptr = 0xFFFF;
671 th->urg = 1;
672 }
David S. Millerdfb4b9d2005-12-06 16:24:52 -0800673 }
674
Adam Langley33ad7982008-07-19 00:04:31 -0700675 tcp_options_write((__be32 *)(th + 1), tp, &opts, &md5_hash_location);
676 if (likely((tcb->flags & TCPCB_FLAG_SYN) == 0))
Ilpo Järvinen9e412ba2007-04-20 22:18:02 -0700677 TCP_ECN_send(sk, skb, tcp_header_size);
David S. Millerdfb4b9d2005-12-06 16:24:52 -0800678
YOSHIFUJI Hideakicfb6eeb2006-11-14 19:07:45 -0800679#ifdef CONFIG_TCP_MD5SIG
680 /* Calculate the MD5 hash, as we have all we need now */
681 if (md5) {
Adam Langley33ad7982008-07-19 00:04:31 -0700682 sk->sk_route_caps &= ~NETIF_F_GSO_MASK;
YOSHIFUJI Hideakicfb6eeb2006-11-14 19:07:45 -0800683 tp->af_specific->calc_md5_hash(md5_hash_location,
Adam Langley49a72df2008-07-19 00:01:42 -0700684 md5, sk, NULL, skb);
YOSHIFUJI Hideakicfb6eeb2006-11-14 19:07:45 -0800685 }
686#endif
687
Arnaldo Carvalho de Melo8292a172005-12-13 23:15:52 -0800688 icsk->icsk_af_ops->send_check(sk, skb->len, skb);
David S. Millerdfb4b9d2005-12-06 16:24:52 -0800689
690 if (likely(tcb->flags & TCPCB_FLAG_ACK))
691 tcp_event_ack_sent(sk, tcp_skb_pcount(skb));
692
693 if (skb->len != tcp_header_size)
694 tcp_event_data_sent(tp, skb, sk);
695
Wei Yongjunbd37a082006-08-07 21:04:15 -0700696 if (after(tcb->end_seq, tp->snd_nxt) || tcb->seq == tcb->end_seq)
Pavel Emelyanov81cc8a72008-07-16 20:22:04 -0700697 TCP_INC_STATS(sock_net(sk), TCP_MIB_OUTSEGS);
David S. Millerdfb4b9d2005-12-06 16:24:52 -0800698
David S. Millere89862f2007-01-26 01:04:55 -0800699 err = icsk->icsk_af_ops->queue_xmit(skb, 0);
Hua Zhong83de47c2006-04-28 15:26:50 -0700700 if (likely(err <= 0))
David S. Millerdfb4b9d2005-12-06 16:24:52 -0800701 return err;
702
Ilpo Järvinen3cfe3ba2007-02-27 10:09:49 -0800703 tcp_enter_cwr(sk, 1);
David S. Millerdfb4b9d2005-12-06 16:24:52 -0800704
Gerrit Renkerb9df3cb2006-11-14 11:21:36 -0200705 return net_xmit_eval(err);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700706}
707
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900708/* This routine just queue's the buffer
Linus Torvalds1da177e2005-04-16 15:20:36 -0700709 *
710 * NOTE: probe0 timer is not checked, do not forget tcp_push_pending_frames,
711 * otherwise socket can stall.
712 */
713static void tcp_queue_skb(struct sock *sk, struct sk_buff *skb)
714{
715 struct tcp_sock *tp = tcp_sk(sk);
716
717 /* Advance write_seq and place onto the write_queue. */
718 tp->write_seq = TCP_SKB_CB(skb)->end_seq;
719 skb_header_release(skb);
David S. Millerfe067e82007-03-07 12:12:44 -0800720 tcp_add_write_queue_tail(sk, skb);
Hideo Aoki3ab224b2007-12-31 00:11:19 -0800721 sk->sk_wmem_queued += skb->truesize;
722 sk_mem_charge(sk, skb->truesize);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700723}
724
Ilpo Järvinen056834d2007-12-31 14:57:14 -0800725static void tcp_set_skb_tso_segs(struct sock *sk, struct sk_buff *skb,
726 unsigned int mss_now)
David S. Millerf6302d12005-07-05 15:18:03 -0700727{
Ilpo Järvinenf8269a42008-12-03 21:24:48 -0800728 if (skb->len <= mss_now || !sk_can_gso(sk)) {
David S. Millerf6302d12005-07-05 15:18:03 -0700729 /* Avoid the costly divide in the normal
730 * non-TSO case.
731 */
Herbert Xu79671682006-06-22 02:40:14 -0700732 skb_shinfo(skb)->gso_segs = 1;
733 skb_shinfo(skb)->gso_size = 0;
734 skb_shinfo(skb)->gso_type = 0;
David S. Millerf6302d12005-07-05 15:18:03 -0700735 } else {
Ilpo Järvinen356f89e2007-08-24 23:00:31 -0700736 skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(skb->len, mss_now);
Herbert Xu79671682006-06-22 02:40:14 -0700737 skb_shinfo(skb)->gso_size = mss_now;
Herbert Xubcd76112006-06-30 13:36:35 -0700738 skb_shinfo(skb)->gso_type = sk->sk_gso_type;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700739 }
740}
741
Ilpo Järvinen91fed7a2007-10-09 01:24:15 -0700742/* When a modification to fackets out becomes necessary, we need to check
Ilpo Järvinen68f83532007-11-15 19:50:37 -0800743 * skb is counted to fackets_out or not.
Ilpo Järvinen91fed7a2007-10-09 01:24:15 -0700744 */
Ilpo Järvinena47e5a92007-11-15 19:41:46 -0800745static void tcp_adjust_fackets_out(struct sock *sk, struct sk_buff *skb,
Ilpo Järvinen91fed7a2007-10-09 01:24:15 -0700746 int decr)
747{
Ilpo Järvinena47e5a92007-11-15 19:41:46 -0800748 struct tcp_sock *tp = tcp_sk(sk);
749
Ilpo Järvinendc869672007-10-01 15:27:19 -0700750 if (!tp->sacked_out || tcp_is_reno(tp))
Ilpo Järvinen91fed7a2007-10-09 01:24:15 -0700751 return;
752
Ilpo Järvinen6859d492007-12-02 00:48:06 +0200753 if (after(tcp_highest_sack_seq(tp), TCP_SKB_CB(skb)->seq))
Ilpo Järvinen91fed7a2007-10-09 01:24:15 -0700754 tp->fackets_out -= decr;
Ilpo Järvinen91fed7a2007-10-09 01:24:15 -0700755}
756
Ilpo Järvinen797108d2009-04-01 23:15:17 +0000757/* Pcount in the middle of the write queue got changed, we need to do various
758 * tweaks to fix counters
759 */
760static void tcp_adjust_pcount(struct sock *sk, struct sk_buff *skb, int decr)
761{
762 struct tcp_sock *tp = tcp_sk(sk);
763
764 tp->packets_out -= decr;
765
766 if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)
767 tp->sacked_out -= decr;
768 if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS)
769 tp->retrans_out -= decr;
770 if (TCP_SKB_CB(skb)->sacked & TCPCB_LOST)
771 tp->lost_out -= decr;
772
773 /* Reno case is special. Sigh... */
774 if (tcp_is_reno(tp) && decr > 0)
775 tp->sacked_out -= min_t(u32, tp->sacked_out, decr);
776
777 tcp_adjust_fackets_out(sk, skb, decr);
778
779 if (tp->lost_skb_hint &&
780 before(TCP_SKB_CB(skb)->seq, TCP_SKB_CB(tp->lost_skb_hint)->seq) &&
Ilpo Järvinen52cf3cc2009-04-18 05:48:48 +0000781 (tcp_is_fack(tp) || (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)))
Ilpo Järvinen797108d2009-04-01 23:15:17 +0000782 tp->lost_cnt_hint -= decr;
783
784 tcp_verify_left_out(tp);
785}
786
Linus Torvalds1da177e2005-04-16 15:20:36 -0700787/* Function to create two new TCP segments. Shrinks the given segment
788 * to the specified size and appends a new segment with the rest of the
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900789 * packet to the list. This won't be called frequently, I hope.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700790 * Remember, these are still headerless SKBs at this point.
791 */
Ilpo Järvinen056834d2007-12-31 14:57:14 -0800792int tcp_fragment(struct sock *sk, struct sk_buff *skb, u32 len,
793 unsigned int mss_now)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700794{
795 struct tcp_sock *tp = tcp_sk(sk);
796 struct sk_buff *buff;
David S. Miller6475be12005-09-01 22:47:01 -0700797 int nsize, old_factor;
Herbert Xub60b49e2006-04-19 21:35:00 -0700798 int nlen;
Ilpo Järvinen9ce01462009-02-28 04:44:42 +0000799 u8 flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700800
Herbert Xub2cc99f02005-10-20 17:13:13 -0200801 BUG_ON(len > skb->len);
Stephen Hemminger6a438bb2005-11-10 17:14:59 -0800802
Linus Torvalds1da177e2005-04-16 15:20:36 -0700803 nsize = skb_headlen(skb) - len;
804 if (nsize < 0)
805 nsize = 0;
806
807 if (skb_cloned(skb) &&
808 skb_is_nonlinear(skb) &&
809 pskb_expand_head(skb, 0, 0, GFP_ATOMIC))
810 return -ENOMEM;
811
812 /* Get a new skb... force flag on. */
813 buff = sk_stream_alloc_skb(sk, nsize, GFP_ATOMIC);
814 if (buff == NULL)
815 return -ENOMEM; /* We'll just try again later. */
Herbert Xuef5cb972006-04-18 13:24:14 -0700816
Hideo Aoki3ab224b2007-12-31 00:11:19 -0800817 sk->sk_wmem_queued += buff->truesize;
818 sk_mem_charge(sk, buff->truesize);
Herbert Xub60b49e2006-04-19 21:35:00 -0700819 nlen = skb->len - len - nsize;
820 buff->truesize += nlen;
821 skb->truesize -= nlen;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700822
823 /* Correct the sequence numbers. */
824 TCP_SKB_CB(buff)->seq = TCP_SKB_CB(skb)->seq + len;
825 TCP_SKB_CB(buff)->end_seq = TCP_SKB_CB(skb)->end_seq;
826 TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(buff)->seq;
827
828 /* PSH and FIN should only be set in the second packet. */
829 flags = TCP_SKB_CB(skb)->flags;
Ilpo Järvinen056834d2007-12-31 14:57:14 -0800830 TCP_SKB_CB(skb)->flags = flags & ~(TCPCB_FLAG_FIN | TCPCB_FLAG_PSH);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700831 TCP_SKB_CB(buff)->flags = flags;
Herbert Xue14c3ca2005-09-19 18:18:38 -0700832 TCP_SKB_CB(buff)->sacked = TCP_SKB_CB(skb)->sacked;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700833
Patrick McHardy84fa7932006-08-29 16:44:56 -0700834 if (!skb_shinfo(skb)->nr_frags && skb->ip_summed != CHECKSUM_PARTIAL) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700835 /* Copy and checksum data tail into the new buffer. */
Ilpo Järvinen056834d2007-12-31 14:57:14 -0800836 buff->csum = csum_partial_copy_nocheck(skb->data + len,
837 skb_put(buff, nsize),
Linus Torvalds1da177e2005-04-16 15:20:36 -0700838 nsize, 0);
839
840 skb_trim(skb, len);
841
842 skb->csum = csum_block_sub(skb->csum, buff->csum, len);
843 } else {
Patrick McHardy84fa7932006-08-29 16:44:56 -0700844 skb->ip_summed = CHECKSUM_PARTIAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700845 skb_split(skb, buff, len);
846 }
847
848 buff->ip_summed = skb->ip_summed;
849
850 /* Looks stupid, but our code really uses when of
851 * skbs, which it never sent before. --ANK
852 */
853 TCP_SKB_CB(buff)->when = TCP_SKB_CB(skb)->when;
Patrick McHardya61bbcf2005-08-14 17:24:31 -0700854 buff->tstamp = skb->tstamp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700855
David S. Miller6475be12005-09-01 22:47:01 -0700856 old_factor = tcp_skb_pcount(skb);
857
Linus Torvalds1da177e2005-04-16 15:20:36 -0700858 /* Fix up tso_factor for both original and new SKB. */
David S. Miller846998a2005-08-04 19:52:01 -0700859 tcp_set_skb_tso_segs(sk, skb, mss_now);
860 tcp_set_skb_tso_segs(sk, buff, mss_now);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700861
David S. Miller6475be12005-09-01 22:47:01 -0700862 /* If this packet has been sent out already, we must
863 * adjust the various packet counters.
864 */
Herbert Xucf0b4502005-09-08 15:10:52 -0700865 if (!before(tp->snd_nxt, TCP_SKB_CB(buff)->end_seq)) {
David S. Miller6475be12005-09-01 22:47:01 -0700866 int diff = old_factor - tcp_skb_pcount(skb) -
867 tcp_skb_pcount(buff);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700868
Ilpo Järvinen797108d2009-04-01 23:15:17 +0000869 if (diff)
870 tcp_adjust_pcount(sk, skb, diff);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700871 }
872
873 /* Link BUFF into the send queue. */
David S. Millerf44b5272005-07-05 15:18:34 -0700874 skb_header_release(buff);
David S. Millerfe067e82007-03-07 12:12:44 -0800875 tcp_insert_write_queue_after(skb, buff, sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700876
877 return 0;
878}
879
880/* This is similar to __pskb_pull_head() (it will go to core/skbuff.c
881 * eventually). The difference is that pulled data not copied, but
882 * immediately discarded.
883 */
Herbert Xu ~{PmVHI~}f2911962006-06-05 15:03:37 -0700884static void __pskb_trim_head(struct sk_buff *skb, int len)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700885{
886 int i, k, eat;
887
888 eat = len;
889 k = 0;
Ilpo Järvinen056834d2007-12-31 14:57:14 -0800890 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700891 if (skb_shinfo(skb)->frags[i].size <= eat) {
892 put_page(skb_shinfo(skb)->frags[i].page);
893 eat -= skb_shinfo(skb)->frags[i].size;
894 } else {
895 skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i];
896 if (eat) {
897 skb_shinfo(skb)->frags[k].page_offset += eat;
898 skb_shinfo(skb)->frags[k].size -= eat;
899 eat = 0;
900 }
901 k++;
902 }
903 }
904 skb_shinfo(skb)->nr_frags = k;
905
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -0700906 skb_reset_tail_pointer(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700907 skb->data_len -= len;
908 skb->len = skb->data_len;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700909}
910
911int tcp_trim_head(struct sock *sk, struct sk_buff *skb, u32 len)
912{
Ilpo Järvinen056834d2007-12-31 14:57:14 -0800913 if (skb_cloned(skb) && pskb_expand_head(skb, 0, 0, GFP_ATOMIC))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700914 return -ENOMEM;
915
Herbert Xu ~{PmVHI~}f2911962006-06-05 15:03:37 -0700916 /* If len == headlen, we avoid __skb_pull to preserve alignment. */
917 if (unlikely(len < skb_headlen(skb)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700918 __skb_pull(skb, len);
Herbert Xu ~{PmVHI~}f2911962006-06-05 15:03:37 -0700919 else
920 __pskb_trim_head(skb, len - skb_headlen(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700921
922 TCP_SKB_CB(skb)->seq += len;
Patrick McHardy84fa7932006-08-29 16:44:56 -0700923 skb->ip_summed = CHECKSUM_PARTIAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700924
925 skb->truesize -= len;
926 sk->sk_wmem_queued -= len;
Hideo Aoki3ab224b2007-12-31 00:11:19 -0800927 sk_mem_uncharge(sk, len);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700928 sock_set_flag(sk, SOCK_QUEUE_SHRUNK);
929
930 /* Any change of skb->len requires recalculation of tso
931 * factor and mss.
932 */
933 if (tcp_skb_pcount(skb) > 1)
Ilpo Järvinen0c54b852009-03-14 14:23:05 +0000934 tcp_set_skb_tso_segs(sk, skb, tcp_current_mss(sk));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700935
936 return 0;
937}
938
John Heffner5d424d52006-03-20 17:53:41 -0800939/* Not accounting for SACKs here. */
940int tcp_mtu_to_mss(struct sock *sk, int pmtu)
941{
942 struct tcp_sock *tp = tcp_sk(sk);
943 struct inet_connection_sock *icsk = inet_csk(sk);
944 int mss_now;
945
946 /* Calculate base mss without TCP options:
947 It is MMS_S - sizeof(tcphdr) of rfc1122
948 */
949 mss_now = pmtu - icsk->icsk_af_ops->net_header_len - sizeof(struct tcphdr);
950
951 /* Clamp it (mss_clamp does not include tcp options) */
952 if (mss_now > tp->rx_opt.mss_clamp)
953 mss_now = tp->rx_opt.mss_clamp;
954
955 /* Now subtract optional transport overhead */
956 mss_now -= icsk->icsk_ext_hdr_len;
957
958 /* Then reserve room for full set of TCP options and 8 bytes of data */
959 if (mss_now < 48)
960 mss_now = 48;
961
962 /* Now subtract TCP options size, not including SACKs */
963 mss_now -= tp->tcp_header_len - sizeof(struct tcphdr);
964
965 return mss_now;
966}
967
968/* Inverse of above */
969int tcp_mss_to_mtu(struct sock *sk, int mss)
970{
971 struct tcp_sock *tp = tcp_sk(sk);
972 struct inet_connection_sock *icsk = inet_csk(sk);
973 int mtu;
974
975 mtu = mss +
976 tp->tcp_header_len +
977 icsk->icsk_ext_hdr_len +
978 icsk->icsk_af_ops->net_header_len;
979
980 return mtu;
981}
982
983void tcp_mtup_init(struct sock *sk)
984{
985 struct tcp_sock *tp = tcp_sk(sk);
986 struct inet_connection_sock *icsk = inet_csk(sk);
987
988 icsk->icsk_mtup.enabled = sysctl_tcp_mtu_probing > 1;
989 icsk->icsk_mtup.search_high = tp->rx_opt.mss_clamp + sizeof(struct tcphdr) +
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +0900990 icsk->icsk_af_ops->net_header_len;
John Heffner5d424d52006-03-20 17:53:41 -0800991 icsk->icsk_mtup.search_low = tcp_mss_to_mtu(sk, sysctl_tcp_base_mss);
992 icsk->icsk_mtup.probe_size = 0;
993}
994
Linus Torvalds1da177e2005-04-16 15:20:36 -0700995/* This function synchronize snd mss to current pmtu/exthdr set.
996
997 tp->rx_opt.user_mss is mss set by user by TCP_MAXSEG. It does NOT counts
998 for TCP options, but includes only bare TCP header.
999
1000 tp->rx_opt.mss_clamp is mss negotiated at connection setup.
Stephen Hemmingercaa20d9a2005-11-10 17:13:47 -08001001 It is minimum of user_mss and mss received with SYN.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001002 It also does not include TCP options.
1003
Arnaldo Carvalho de Melod83d8462005-12-13 23:26:10 -08001004 inet_csk(sk)->icsk_pmtu_cookie is last pmtu, seen by this function.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001005
1006 tp->mss_cache is current effective sending mss, including
1007 all tcp options except for SACKs. It is evaluated,
1008 taking into account current pmtu, but never exceeds
1009 tp->rx_opt.mss_clamp.
1010
1011 NOTE1. rfc1122 clearly states that advertised MSS
1012 DOES NOT include either tcp or ip options.
1013
Arnaldo Carvalho de Melod83d8462005-12-13 23:26:10 -08001014 NOTE2. inet_csk(sk)->icsk_pmtu_cookie and tp->mss_cache
1015 are READ ONLY outside this function. --ANK (980731)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001016 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001017unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu)
1018{
1019 struct tcp_sock *tp = tcp_sk(sk);
Arnaldo Carvalho de Melod83d8462005-12-13 23:26:10 -08001020 struct inet_connection_sock *icsk = inet_csk(sk);
John Heffner5d424d52006-03-20 17:53:41 -08001021 int mss_now;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001022
John Heffner5d424d52006-03-20 17:53:41 -08001023 if (icsk->icsk_mtup.search_high > pmtu)
1024 icsk->icsk_mtup.search_high = pmtu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001025
John Heffner5d424d52006-03-20 17:53:41 -08001026 mss_now = tcp_mtu_to_mss(sk, pmtu);
Ilpo Järvinen409d22b2007-12-31 14:57:40 -08001027 mss_now = tcp_bound_to_half_wnd(tp, mss_now);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001028
1029 /* And store cached results */
Arnaldo Carvalho de Melod83d8462005-12-13 23:26:10 -08001030 icsk->icsk_pmtu_cookie = pmtu;
John Heffner5d424d52006-03-20 17:53:41 -08001031 if (icsk->icsk_mtup.enabled)
1032 mss_now = min(mss_now, tcp_mtu_to_mss(sk, icsk->icsk_mtup.search_low));
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001033 tp->mss_cache = mss_now;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001034
1035 return mss_now;
1036}
1037
1038/* Compute the current effective MSS, taking SACKs and IP options,
1039 * and even PMTU discovery events into account.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001040 */
Ilpo Järvinen0c54b852009-03-14 14:23:05 +00001041unsigned int tcp_current_mss(struct sock *sk)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001042{
1043 struct tcp_sock *tp = tcp_sk(sk);
1044 struct dst_entry *dst = __sk_dst_get(sk);
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001045 u32 mss_now;
Adam Langley33ad7982008-07-19 00:04:31 -07001046 unsigned header_len;
1047 struct tcp_out_options opts;
1048 struct tcp_md5sig_key *md5;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001049
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001050 mss_now = tp->mss_cache;
1051
Linus Torvalds1da177e2005-04-16 15:20:36 -07001052 if (dst) {
1053 u32 mtu = dst_mtu(dst);
Arnaldo Carvalho de Melod83d8462005-12-13 23:26:10 -08001054 if (mtu != inet_csk(sk)->icsk_pmtu_cookie)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001055 mss_now = tcp_sync_mss(sk, mtu);
1056 }
1057
Adam Langley33ad7982008-07-19 00:04:31 -07001058 header_len = tcp_established_options(sk, NULL, &opts, &md5) +
1059 sizeof(struct tcphdr);
1060 /* The mss_cache is sized based on tp->tcp_header_len, which assumes
1061 * some common options. If this is an odd packet (because we have SACK
1062 * blocks etc) then our calculated header_len will be different, and
1063 * we have to adjust mss_now correspondingly */
1064 if (header_len != tp->tcp_header_len) {
1065 int delta = (int) header_len - tp->tcp_header_len;
1066 mss_now -= delta;
1067 }
YOSHIFUJI Hideakicfb6eeb2006-11-14 19:07:45 -08001068
Linus Torvalds1da177e2005-04-16 15:20:36 -07001069 return mss_now;
1070}
1071
David S. Millera762a982005-07-05 15:18:51 -07001072/* Congestion window validation. (RFC2861) */
Ilpo Järvinen9e412ba2007-04-20 22:18:02 -07001073static void tcp_cwnd_validate(struct sock *sk)
David S. Millera762a982005-07-05 15:18:51 -07001074{
Ilpo Järvinen9e412ba2007-04-20 22:18:02 -07001075 struct tcp_sock *tp = tcp_sk(sk);
David S. Millera762a982005-07-05 15:18:51 -07001076
Ilpo Järvinend436d682007-12-31 14:58:00 -08001077 if (tp->packets_out >= tp->snd_cwnd) {
David S. Millera762a982005-07-05 15:18:51 -07001078 /* Network is feed fully. */
1079 tp->snd_cwnd_used = 0;
1080 tp->snd_cwnd_stamp = tcp_time_stamp;
1081 } else {
1082 /* Network starves. */
1083 if (tp->packets_out > tp->snd_cwnd_used)
1084 tp->snd_cwnd_used = tp->packets_out;
1085
David S. Miller15d33c02007-04-09 13:23:14 -07001086 if (sysctl_tcp_slow_start_after_idle &&
1087 (s32)(tcp_time_stamp - tp->snd_cwnd_stamp) >= inet_csk(sk)->icsk_rto)
David S. Millera762a982005-07-05 15:18:51 -07001088 tcp_cwnd_application_limited(sk);
1089 }
1090}
1091
Ilpo Järvinen0e3a4802007-12-24 21:33:45 -08001092/* Returns the portion of skb which can be sent right away without
1093 * introducing MSS oddities to segment boundaries. In rare cases where
1094 * mss_now != mss_cache, we will request caller to create a small skb
1095 * per input skb which could be mostly avoided here (if desired).
Ilpo Järvinen5ea3a742008-03-11 17:55:27 -07001096 *
1097 * We explicitly want to create a request for splitting write queue tail
1098 * to a small skb for Nagle purposes while avoiding unnecessary modulos,
1099 * thus all the complexity (cwnd_len is always MSS multiple which we
1100 * return whenever allowed by the other factors). Basically we need the
1101 * modulo only when the receiver window alone is the limiting factor or
1102 * when we would be allowed to send the split-due-to-Nagle skb fully.
Ilpo Järvinen0e3a4802007-12-24 21:33:45 -08001103 */
1104static unsigned int tcp_mss_split_point(struct sock *sk, struct sk_buff *skb,
Ilpo Järvinen056834d2007-12-31 14:57:14 -08001105 unsigned int mss_now, unsigned int cwnd)
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001106{
Ilpo Järvinen0e3a4802007-12-24 21:33:45 -08001107 struct tcp_sock *tp = tcp_sk(sk);
1108 u32 needed, window, cwnd_len;
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001109
Ilpo Järvinen90840de2007-12-31 04:48:41 -08001110 window = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001111 cwnd_len = mss_now * cwnd;
Ilpo Järvinen0e3a4802007-12-24 21:33:45 -08001112
1113 if (likely(cwnd_len <= window && skb != tcp_write_queue_tail(sk)))
1114 return cwnd_len;
1115
Ilpo Järvinen5ea3a742008-03-11 17:55:27 -07001116 needed = min(skb->len, window);
1117
Ilpo Järvinen17515402008-04-15 20:36:55 -07001118 if (cwnd_len <= needed)
Ilpo Järvinen0e3a4802007-12-24 21:33:45 -08001119 return cwnd_len;
1120
Ilpo Järvinen0e3a4802007-12-24 21:33:45 -08001121 return needed - needed % mss_now;
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001122}
1123
1124/* Can at least one segment of SKB be sent right now, according to the
1125 * congestion window rules? If so, return how many segments are allowed.
1126 */
Ilpo Järvinen056834d2007-12-31 14:57:14 -08001127static inline unsigned int tcp_cwnd_test(struct tcp_sock *tp,
1128 struct sk_buff *skb)
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001129{
1130 u32 in_flight, cwnd;
1131
1132 /* Don't be strict about the congestion window for the final FIN. */
John Heffner104439a2007-02-05 17:53:11 -08001133 if ((TCP_SKB_CB(skb)->flags & TCPCB_FLAG_FIN) &&
1134 tcp_skb_pcount(skb) == 1)
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001135 return 1;
1136
1137 in_flight = tcp_packets_in_flight(tp);
1138 cwnd = tp->snd_cwnd;
1139 if (in_flight < cwnd)
1140 return (cwnd - in_flight);
1141
1142 return 0;
1143}
1144
1145/* This must be invoked the first time we consider transmitting
1146 * SKB onto the wire.
1147 */
Ilpo Järvinen056834d2007-12-31 14:57:14 -08001148static int tcp_init_tso_segs(struct sock *sk, struct sk_buff *skb,
1149 unsigned int mss_now)
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001150{
1151 int tso_segs = tcp_skb_pcount(skb);
1152
Ilpo Järvinenf8269a42008-12-03 21:24:48 -08001153 if (!tso_segs || (tso_segs > 1 && tcp_skb_mss(skb) != mss_now)) {
David S. Miller846998a2005-08-04 19:52:01 -07001154 tcp_set_skb_tso_segs(sk, skb, mss_now);
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001155 tso_segs = tcp_skb_pcount(skb);
1156 }
1157 return tso_segs;
1158}
1159
1160static inline int tcp_minshall_check(const struct tcp_sock *tp)
1161{
Jianjun Kong09cb1052008-11-03 00:27:11 -08001162 return after(tp->snd_sml, tp->snd_una) &&
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001163 !after(tp->snd_sml, tp->snd_nxt);
1164}
1165
1166/* Return 0, if packet can be sent now without violation Nagle's rules:
1167 * 1. It is full sized.
1168 * 2. Or it contains FIN. (already checked by caller)
1169 * 3. Or TCP_NODELAY was set.
1170 * 4. Or TCP_CORK is not set, and all sent packets are ACKed.
1171 * With Minshall's modification: all sent small packets are ACKed.
1172 */
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001173static inline int tcp_nagle_check(const struct tcp_sock *tp,
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09001174 const struct sk_buff *skb,
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001175 unsigned mss_now, int nonagle)
1176{
1177 return (skb->len < mss_now &&
Ilpo Järvinen056834d2007-12-31 14:57:14 -08001178 ((nonagle & TCP_NAGLE_CORK) ||
1179 (!nonagle && tp->packets_out && tcp_minshall_check(tp))));
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001180}
1181
1182/* Return non-zero if the Nagle test allows this packet to be
1183 * sent now.
1184 */
1185static inline int tcp_nagle_test(struct tcp_sock *tp, struct sk_buff *skb,
1186 unsigned int cur_mss, int nonagle)
1187{
1188 /* Nagle rule does not apply to frames, which sit in the middle of the
1189 * write_queue (they have no chances to get new data).
1190 *
1191 * This is implemented in the callers, where they modify the 'nonagle'
1192 * argument based upon the location of SKB in the send queue.
1193 */
1194 if (nonagle & TCP_NAGLE_PUSH)
1195 return 1;
1196
Ilpo Järvinend551e452007-04-30 00:42:20 -07001197 /* Don't use the nagle rule for urgent data (or for the final FIN).
1198 * Nagle can be ignored during F-RTO too (see RFC4138).
1199 */
Ilpo Järvinen33f5f572008-10-07 14:43:06 -07001200 if (tcp_urg_mode(tp) || (tp->frto_counter == 2) ||
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001201 (TCP_SKB_CB(skb)->flags & TCPCB_FLAG_FIN))
1202 return 1;
1203
1204 if (!tcp_nagle_check(tp, skb, cur_mss, nonagle))
1205 return 1;
1206
1207 return 0;
1208}
1209
1210/* Does at least the first segment of SKB fit into the send window? */
Ilpo Järvinen056834d2007-12-31 14:57:14 -08001211static inline int tcp_snd_wnd_test(struct tcp_sock *tp, struct sk_buff *skb,
1212 unsigned int cur_mss)
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001213{
1214 u32 end_seq = TCP_SKB_CB(skb)->end_seq;
1215
1216 if (skb->len > cur_mss)
1217 end_seq = TCP_SKB_CB(skb)->seq + cur_mss;
1218
Ilpo Järvinen90840de2007-12-31 04:48:41 -08001219 return !after(end_seq, tcp_wnd_end(tp));
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001220}
1221
David S. Millerfe067e82007-03-07 12:12:44 -08001222/* This checks if the data bearing packet SKB (usually tcp_send_head(sk))
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001223 * should be put on the wire right now. If so, it returns the number of
1224 * packets allowed by the congestion window.
1225 */
1226static unsigned int tcp_snd_test(struct sock *sk, struct sk_buff *skb,
1227 unsigned int cur_mss, int nonagle)
1228{
1229 struct tcp_sock *tp = tcp_sk(sk);
1230 unsigned int cwnd_quota;
1231
David S. Miller846998a2005-08-04 19:52:01 -07001232 tcp_init_tso_segs(sk, skb, cur_mss);
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001233
1234 if (!tcp_nagle_test(tp, skb, cur_mss, nonagle))
1235 return 0;
1236
1237 cwnd_quota = tcp_cwnd_test(tp, skb);
Ilpo Järvinen056834d2007-12-31 14:57:14 -08001238 if (cwnd_quota && !tcp_snd_wnd_test(tp, skb, cur_mss))
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001239 cwnd_quota = 0;
1240
1241 return cwnd_quota;
1242}
1243
Ilpo Järvinen9e412ba2007-04-20 22:18:02 -07001244int tcp_may_send_now(struct sock *sk)
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001245{
Ilpo Järvinen9e412ba2007-04-20 22:18:02 -07001246 struct tcp_sock *tp = tcp_sk(sk);
David S. Millerfe067e82007-03-07 12:12:44 -08001247 struct sk_buff *skb = tcp_send_head(sk);
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001248
1249 return (skb &&
Ilpo Järvinen0c54b852009-03-14 14:23:05 +00001250 tcp_snd_test(sk, skb, tcp_current_mss(sk),
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001251 (tcp_skb_is_last(sk, skb) ?
Ilpo Järvinen4e67d872007-12-05 02:25:32 -08001252 tp->nonagle : TCP_NAGLE_PUSH)));
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001253}
1254
1255/* Trim TSO SKB to LEN bytes, put the remaining data into a new packet
1256 * which is put after SKB on the list. It is very much like
1257 * tcp_fragment() except that it may make several kinds of assumptions
1258 * in order to speed up the splitting operation. In particular, we
1259 * know that all the data is in scatter-gather pages, and that the
1260 * packet has never been sent out before (and thus is not cloned).
1261 */
Ilpo Järvinen056834d2007-12-31 14:57:14 -08001262static int tso_fragment(struct sock *sk, struct sk_buff *skb, unsigned int len,
1263 unsigned int mss_now)
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001264{
1265 struct sk_buff *buff;
1266 int nlen = skb->len - len;
Ilpo Järvinen9ce01462009-02-28 04:44:42 +00001267 u8 flags;
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001268
1269 /* All of a TSO frame must be composed of paged data. */
Herbert Xuc8ac3772005-08-16 20:43:40 -07001270 if (skb->len != skb->data_len)
1271 return tcp_fragment(sk, skb, len, mss_now);
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001272
Pavel Emelyanovdf97c702007-11-29 21:22:33 +11001273 buff = sk_stream_alloc_skb(sk, 0, GFP_ATOMIC);
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001274 if (unlikely(buff == NULL))
1275 return -ENOMEM;
1276
Hideo Aoki3ab224b2007-12-31 00:11:19 -08001277 sk->sk_wmem_queued += buff->truesize;
1278 sk_mem_charge(sk, buff->truesize);
Herbert Xub60b49e2006-04-19 21:35:00 -07001279 buff->truesize += nlen;
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001280 skb->truesize -= nlen;
1281
1282 /* Correct the sequence numbers. */
1283 TCP_SKB_CB(buff)->seq = TCP_SKB_CB(skb)->seq + len;
1284 TCP_SKB_CB(buff)->end_seq = TCP_SKB_CB(skb)->end_seq;
1285 TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(buff)->seq;
1286
1287 /* PSH and FIN should only be set in the second packet. */
1288 flags = TCP_SKB_CB(skb)->flags;
Ilpo Järvinen056834d2007-12-31 14:57:14 -08001289 TCP_SKB_CB(skb)->flags = flags & ~(TCPCB_FLAG_FIN | TCPCB_FLAG_PSH);
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001290 TCP_SKB_CB(buff)->flags = flags;
1291
1292 /* This packet was never sent out yet, so no SACK bits. */
1293 TCP_SKB_CB(buff)->sacked = 0;
1294
Patrick McHardy84fa7932006-08-29 16:44:56 -07001295 buff->ip_summed = skb->ip_summed = CHECKSUM_PARTIAL;
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001296 skb_split(skb, buff, len);
1297
1298 /* Fix up tso_factor for both original and new SKB. */
David S. Miller846998a2005-08-04 19:52:01 -07001299 tcp_set_skb_tso_segs(sk, skb, mss_now);
1300 tcp_set_skb_tso_segs(sk, buff, mss_now);
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001301
1302 /* Link BUFF into the send queue. */
1303 skb_header_release(buff);
David S. Millerfe067e82007-03-07 12:12:44 -08001304 tcp_insert_write_queue_after(skb, buff, sk);
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001305
1306 return 0;
1307}
1308
1309/* Try to defer sending, if possible, in order to minimize the amount
1310 * of TSO splitting we do. View it as a kind of TSO Nagle test.
1311 *
1312 * This algorithm is from John Heffner.
1313 */
Ilpo Järvinen9e412ba2007-04-20 22:18:02 -07001314static int tcp_tso_should_defer(struct sock *sk, struct sk_buff *skb)
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001315{
Ilpo Järvinen9e412ba2007-04-20 22:18:02 -07001316 struct tcp_sock *tp = tcp_sk(sk);
Arnaldo Carvalho de Melo6687e982005-08-10 04:03:31 -03001317 const struct inet_connection_sock *icsk = inet_csk(sk);
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001318 u32 send_win, cong_win, limit, in_flight;
1319
1320 if (TCP_SKB_CB(skb)->flags & TCPCB_FLAG_FIN)
John Heffnerae8064a2006-10-18 20:36:48 -07001321 goto send_now;
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001322
Arnaldo Carvalho de Melo6687e982005-08-10 04:03:31 -03001323 if (icsk->icsk_ca_state != TCP_CA_Open)
John Heffnerae8064a2006-10-18 20:36:48 -07001324 goto send_now;
1325
1326 /* Defer for less than two clock ticks. */
Ilpo Järvinenbd515c32007-12-20 20:36:03 -08001327 if (tp->tso_deferred &&
Ilpo Järvinena2acde02008-12-05 22:49:37 -08001328 (((u32)jiffies << 1) >> 1) - (tp->tso_deferred >> 1) > 1)
John Heffnerae8064a2006-10-18 20:36:48 -07001329 goto send_now;
David S. Miller908a75c2005-07-05 15:43:58 -07001330
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001331 in_flight = tcp_packets_in_flight(tp);
1332
Ilpo Järvinen056834d2007-12-31 14:57:14 -08001333 BUG_ON(tcp_skb_pcount(skb) <= 1 || (tp->snd_cwnd <= in_flight));
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001334
Ilpo Järvinen90840de2007-12-31 04:48:41 -08001335 send_win = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001336
1337 /* From in_flight test above, we know that cwnd > in_flight. */
1338 cong_win = (tp->snd_cwnd - in_flight) * tp->mss_cache;
1339
1340 limit = min(send_win, cong_win);
1341
David S. Millerba244fe2006-03-11 18:51:49 -08001342 /* If a full-sized TSO skb can be sent, do it. */
Peter P Waskiewicz Jr82cc1a72008-03-21 03:43:19 -07001343 if (limit >= sk->sk_gso_max_size)
John Heffnerae8064a2006-10-18 20:36:48 -07001344 goto send_now;
David S. Millerba244fe2006-03-11 18:51:49 -08001345
Ilpo Järvinen62ad2762009-02-28 04:44:29 +00001346 /* Middle in queue won't get any more data, full sendable already? */
1347 if ((skb != tcp_write_queue_tail(sk)) && (limit >= skb->len))
1348 goto send_now;
1349
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001350 if (sysctl_tcp_tso_win_divisor) {
1351 u32 chunk = min(tp->snd_wnd, tp->snd_cwnd * tp->mss_cache);
1352
1353 /* If at least some fraction of a window is available,
1354 * just use it.
1355 */
1356 chunk /= sysctl_tcp_tso_win_divisor;
1357 if (limit >= chunk)
John Heffnerae8064a2006-10-18 20:36:48 -07001358 goto send_now;
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001359 } else {
1360 /* Different approach, try not to defer past a single
1361 * ACK. Receiver should ACK every other full sized
1362 * frame, so if we have space for more than 3 frames
1363 * then send now.
1364 */
1365 if (limit > tcp_max_burst(tp) * tp->mss_cache)
John Heffnerae8064a2006-10-18 20:36:48 -07001366 goto send_now;
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001367 }
1368
1369 /* Ok, it looks like it is advisable to defer. */
Ilpo Järvinen056834d2007-12-31 14:57:14 -08001370 tp->tso_deferred = 1 | (jiffies << 1);
John Heffnerae8064a2006-10-18 20:36:48 -07001371
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001372 return 1;
John Heffnerae8064a2006-10-18 20:36:48 -07001373
1374send_now:
1375 tp->tso_deferred = 0;
1376 return 0;
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001377}
1378
John Heffner5d424d52006-03-20 17:53:41 -08001379/* Create a new MTU probe if we are ready.
1380 * Returns 0 if we should wait to probe (no cwnd available),
1381 * 1 if a probe was sent,
Ilpo Järvinen056834d2007-12-31 14:57:14 -08001382 * -1 otherwise
1383 */
John Heffner5d424d52006-03-20 17:53:41 -08001384static int tcp_mtu_probe(struct sock *sk)
1385{
1386 struct tcp_sock *tp = tcp_sk(sk);
1387 struct inet_connection_sock *icsk = inet_csk(sk);
1388 struct sk_buff *skb, *nskb, *next;
1389 int len;
1390 int probe_size;
Ilpo Järvinen91cc17c2007-11-23 19:08:16 +08001391 int size_needed;
John Heffner5d424d52006-03-20 17:53:41 -08001392 int copy;
1393 int mss_now;
1394
1395 /* Not currently probing/verifying,
1396 * not in recovery,
1397 * have enough cwnd, and
1398 * not SACKing (the variable headers throw things off) */
1399 if (!icsk->icsk_mtup.enabled ||
1400 icsk->icsk_mtup.probe_size ||
1401 inet_csk(sk)->icsk_ca_state != TCP_CA_Open ||
1402 tp->snd_cwnd < 11 ||
Ilpo Järvinencabeccb2009-02-28 04:44:38 +00001403 tp->rx_opt.num_sacks || tp->rx_opt.dsack)
John Heffner5d424d52006-03-20 17:53:41 -08001404 return -1;
1405
1406 /* Very simple search strategy: just double the MSS. */
Ilpo Järvinen0c54b852009-03-14 14:23:05 +00001407 mss_now = tcp_current_mss(sk);
Ilpo Järvinen056834d2007-12-31 14:57:14 -08001408 probe_size = 2 * tp->mss_cache;
Ilpo Järvinen91cc17c2007-11-23 19:08:16 +08001409 size_needed = probe_size + (tp->reordering + 1) * tp->mss_cache;
John Heffner5d424d52006-03-20 17:53:41 -08001410 if (probe_size > tcp_mtu_to_mss(sk, icsk->icsk_mtup.search_high)) {
1411 /* TODO: set timer for probe_converge_event */
1412 return -1;
1413 }
1414
1415 /* Have enough data in the send queue to probe? */
Ilpo Järvinen7f9c33e2007-11-23 19:10:56 +08001416 if (tp->write_seq - tp->snd_nxt < size_needed)
John Heffner5d424d52006-03-20 17:53:41 -08001417 return -1;
1418
Ilpo Järvinen91cc17c2007-11-23 19:08:16 +08001419 if (tp->snd_wnd < size_needed)
1420 return -1;
Ilpo Järvinen90840de2007-12-31 04:48:41 -08001421 if (after(tp->snd_nxt + size_needed, tcp_wnd_end(tp)))
Ilpo Järvinen91cc17c2007-11-23 19:08:16 +08001422 return 0;
John Heffner5d424d52006-03-20 17:53:41 -08001423
Ilpo Järvinend67c58e2007-12-02 00:48:01 +02001424 /* Do we need to wait to drain cwnd? With none in flight, don't stall */
1425 if (tcp_packets_in_flight(tp) + 2 > tp->snd_cwnd) {
1426 if (!tcp_packets_in_flight(tp))
John Heffner5d424d52006-03-20 17:53:41 -08001427 return -1;
1428 else
1429 return 0;
1430 }
1431
1432 /* We're allowed to probe. Build it now. */
1433 if ((nskb = sk_stream_alloc_skb(sk, probe_size, GFP_ATOMIC)) == NULL)
1434 return -1;
Hideo Aoki3ab224b2007-12-31 00:11:19 -08001435 sk->sk_wmem_queued += nskb->truesize;
1436 sk_mem_charge(sk, nskb->truesize);
John Heffner5d424d52006-03-20 17:53:41 -08001437
David S. Millerfe067e82007-03-07 12:12:44 -08001438 skb = tcp_send_head(sk);
John Heffner5d424d52006-03-20 17:53:41 -08001439
1440 TCP_SKB_CB(nskb)->seq = TCP_SKB_CB(skb)->seq;
1441 TCP_SKB_CB(nskb)->end_seq = TCP_SKB_CB(skb)->seq + probe_size;
1442 TCP_SKB_CB(nskb)->flags = TCPCB_FLAG_ACK;
1443 TCP_SKB_CB(nskb)->sacked = 0;
1444 nskb->csum = 0;
Patrick McHardy84fa7932006-08-29 16:44:56 -07001445 nskb->ip_summed = skb->ip_summed;
John Heffner5d424d52006-03-20 17:53:41 -08001446
Ilpo Järvinen50c48172007-12-02 00:48:00 +02001447 tcp_insert_write_queue_before(nskb, skb, sk);
1448
John Heffner5d424d52006-03-20 17:53:41 -08001449 len = 0;
Ilpo Järvinen234b6862007-12-02 00:48:02 +02001450 tcp_for_write_queue_from_safe(skb, next, sk) {
John Heffner5d424d52006-03-20 17:53:41 -08001451 copy = min_t(int, skb->len, probe_size - len);
1452 if (nskb->ip_summed)
1453 skb_copy_bits(skb, 0, skb_put(nskb, copy), copy);
1454 else
1455 nskb->csum = skb_copy_and_csum_bits(skb, 0,
Ilpo Järvinen056834d2007-12-31 14:57:14 -08001456 skb_put(nskb, copy),
1457 copy, nskb->csum);
John Heffner5d424d52006-03-20 17:53:41 -08001458
1459 if (skb->len <= copy) {
1460 /* We've eaten all the data from this skb.
1461 * Throw it away. */
1462 TCP_SKB_CB(nskb)->flags |= TCP_SKB_CB(skb)->flags;
David S. Millerfe067e82007-03-07 12:12:44 -08001463 tcp_unlink_write_queue(skb, sk);
Hideo Aoki3ab224b2007-12-31 00:11:19 -08001464 sk_wmem_free_skb(sk, skb);
John Heffner5d424d52006-03-20 17:53:41 -08001465 } else {
1466 TCP_SKB_CB(nskb)->flags |= TCP_SKB_CB(skb)->flags &
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09001467 ~(TCPCB_FLAG_FIN|TCPCB_FLAG_PSH);
John Heffner5d424d52006-03-20 17:53:41 -08001468 if (!skb_shinfo(skb)->nr_frags) {
1469 skb_pull(skb, copy);
Patrick McHardy84fa7932006-08-29 16:44:56 -07001470 if (skb->ip_summed != CHECKSUM_PARTIAL)
Ilpo Järvinen056834d2007-12-31 14:57:14 -08001471 skb->csum = csum_partial(skb->data,
1472 skb->len, 0);
John Heffner5d424d52006-03-20 17:53:41 -08001473 } else {
1474 __pskb_trim_head(skb, copy);
1475 tcp_set_skb_tso_segs(sk, skb, mss_now);
1476 }
1477 TCP_SKB_CB(skb)->seq += copy;
1478 }
1479
1480 len += copy;
Ilpo Järvinen234b6862007-12-02 00:48:02 +02001481
1482 if (len >= probe_size)
1483 break;
John Heffner5d424d52006-03-20 17:53:41 -08001484 }
1485 tcp_init_tso_segs(sk, nskb, nskb->len);
1486
1487 /* We're ready to send. If this fails, the probe will
1488 * be resegmented into mss-sized pieces by tcp_write_xmit(). */
1489 TCP_SKB_CB(nskb)->when = tcp_time_stamp;
1490 if (!tcp_transmit_skb(sk, nskb, 1, GFP_ATOMIC)) {
1491 /* Decrement cwnd here because we are sending
Ilpo Järvinen056834d2007-12-31 14:57:14 -08001492 * effectively two packets. */
John Heffner5d424d52006-03-20 17:53:41 -08001493 tp->snd_cwnd--;
Ilpo Järvinen66f5fe62007-12-31 04:43:57 -08001494 tcp_event_new_data_sent(sk, nskb);
John Heffner5d424d52006-03-20 17:53:41 -08001495
1496 icsk->icsk_mtup.probe_size = tcp_mss_to_mtu(sk, nskb->len);
John Heffner0e7b1362006-03-20 21:32:58 -08001497 tp->mtu_probe.probe_seq_start = TCP_SKB_CB(nskb)->seq;
1498 tp->mtu_probe.probe_seq_end = TCP_SKB_CB(nskb)->end_seq;
John Heffner5d424d52006-03-20 17:53:41 -08001499
1500 return 1;
1501 }
1502
1503 return -1;
1504}
1505
Linus Torvalds1da177e2005-04-16 15:20:36 -07001506/* This routine writes packets to the network. It advances the
1507 * send_head. This happens as incoming acks open up the remote
1508 * window for us.
1509 *
Ilpo Järvinenf8269a42008-12-03 21:24:48 -08001510 * LARGESEND note: !tcp_urg_mode is overkill, only frames between
1511 * snd_up-64k-mss .. snd_up cannot be large. However, taking into
1512 * account rare use of URG, this is not a big flaw.
1513 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07001514 * Returns 1, if no segments are in flight and we have queued segments, but
1515 * cannot send anything now because of SWS or another problem.
1516 */
Ilpo Järvinend5dd9172008-12-05 22:48:55 -08001517static int tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
1518 int push_one, gfp_t gfp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001519{
1520 struct tcp_sock *tp = tcp_sk(sk);
David S. Miller92df7b52005-07-05 15:19:06 -07001521 struct sk_buff *skb;
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001522 unsigned int tso_segs, sent_pkts;
1523 int cwnd_quota;
John Heffner5d424d52006-03-20 17:53:41 -08001524 int result;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001525
David S. Miller92df7b52005-07-05 15:19:06 -07001526 sent_pkts = 0;
John Heffner5d424d52006-03-20 17:53:41 -08001527
Ilpo Järvinend5dd9172008-12-05 22:48:55 -08001528 if (!push_one) {
1529 /* Do MTU probing. */
1530 result = tcp_mtu_probe(sk);
1531 if (!result) {
1532 return 0;
1533 } else if (result > 0) {
1534 sent_pkts = 1;
1535 }
John Heffner5d424d52006-03-20 17:53:41 -08001536 }
1537
David S. Millerfe067e82007-03-07 12:12:44 -08001538 while ((skb = tcp_send_head(sk))) {
Herbert Xuc8ac3772005-08-16 20:43:40 -07001539 unsigned int limit;
1540
Herbert Xub68e9f82005-08-04 19:52:02 -07001541 tso_segs = tcp_init_tso_segs(sk, skb, mss_now);
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001542 BUG_ON(!tso_segs);
David S. Milleraa934662005-07-05 15:20:09 -07001543
Herbert Xub68e9f82005-08-04 19:52:02 -07001544 cwnd_quota = tcp_cwnd_test(tp, skb);
1545 if (!cwnd_quota)
1546 break;
1547
1548 if (unlikely(!tcp_snd_wnd_test(tp, skb, mss_now)))
1549 break;
1550
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001551 if (tso_segs == 1) {
1552 if (unlikely(!tcp_nagle_test(tp, skb, mss_now,
1553 (tcp_skb_is_last(sk, skb) ?
1554 nonagle : TCP_NAGLE_PUSH))))
1555 break;
1556 } else {
Ilpo Järvinend5dd9172008-12-05 22:48:55 -08001557 if (!push_one && tcp_tso_should_defer(sk, skb))
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001558 break;
1559 }
David S. Milleraa934662005-07-05 15:20:09 -07001560
Herbert Xuc8ac3772005-08-16 20:43:40 -07001561 limit = mss_now;
Ilpo Järvinenf8269a42008-12-03 21:24:48 -08001562 if (tso_segs > 1 && !tcp_urg_mode(tp))
Ilpo Järvinen0e3a4802007-12-24 21:33:45 -08001563 limit = tcp_mss_split_point(sk, skb, mss_now,
1564 cwnd_quota);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001565
Herbert Xuc8ac3772005-08-16 20:43:40 -07001566 if (skb->len > limit &&
1567 unlikely(tso_fragment(sk, skb, limit, mss_now)))
1568 break;
1569
David S. Miller92df7b52005-07-05 15:19:06 -07001570 TCP_SKB_CB(skb)->when = tcp_time_stamp;
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001571
Ilpo Järvinend5dd9172008-12-05 22:48:55 -08001572 if (unlikely(tcp_transmit_skb(sk, skb, 1, gfp)))
David S. Miller92df7b52005-07-05 15:19:06 -07001573 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001574
David S. Miller92df7b52005-07-05 15:19:06 -07001575 /* Advance the send_head. This one is sent out.
1576 * This call will increment packets_out.
1577 */
Ilpo Järvinen66f5fe62007-12-31 04:43:57 -08001578 tcp_event_new_data_sent(sk, skb);
David S. Miller92df7b52005-07-05 15:19:06 -07001579
1580 tcp_minshall_update(tp, mss_now, skb);
David S. Milleraa934662005-07-05 15:20:09 -07001581 sent_pkts++;
Ilpo Järvinend5dd9172008-12-05 22:48:55 -08001582
1583 if (push_one)
1584 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001585 }
David S. Miller92df7b52005-07-05 15:19:06 -07001586
David S. Milleraa934662005-07-05 15:20:09 -07001587 if (likely(sent_pkts)) {
Ilpo Järvinen9e412ba2007-04-20 22:18:02 -07001588 tcp_cwnd_validate(sk);
David S. Miller92df7b52005-07-05 15:19:06 -07001589 return 0;
1590 }
David S. Millerfe067e82007-03-07 12:12:44 -08001591 return !tp->packets_out && tcp_send_head(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001592}
1593
David S. Millera762a982005-07-05 15:18:51 -07001594/* Push out any pending frames which were held back due to
1595 * TCP_CORK or attempt at coalescing tiny packets.
1596 * The socket must be locked by the caller.
1597 */
Ilpo Järvinen9e412ba2007-04-20 22:18:02 -07001598void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss,
1599 int nonagle)
David S. Millera762a982005-07-05 15:18:51 -07001600{
David S. Millerfe067e82007-03-07 12:12:44 -08001601 struct sk_buff *skb = tcp_send_head(sk);
David S. Millera762a982005-07-05 15:18:51 -07001602
Ilpo Järvinen726e07a2008-12-05 22:43:56 -08001603 if (!skb)
1604 return;
1605
1606 /* If we are closed, the bytes will have to remain here.
1607 * In time closedown will finish, we empty the write queue and
1608 * all will be happy.
1609 */
1610 if (unlikely(sk->sk_state == TCP_CLOSE))
1611 return;
1612
Ilpo Järvinend5dd9172008-12-05 22:48:55 -08001613 if (tcp_write_xmit(sk, cur_mss, nonagle, 0, GFP_ATOMIC))
Ilpo Järvinen726e07a2008-12-05 22:43:56 -08001614 tcp_check_probe_timer(sk);
David S. Millera762a982005-07-05 15:18:51 -07001615}
1616
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001617/* Send _single_ skb sitting at the send head. This function requires
1618 * true push pending frames to setup probe timer etc.
1619 */
1620void tcp_push_one(struct sock *sk, unsigned int mss_now)
1621{
David S. Millerfe067e82007-03-07 12:12:44 -08001622 struct sk_buff *skb = tcp_send_head(sk);
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001623
1624 BUG_ON(!skb || skb->len < mss_now);
1625
Ilpo Järvinend5dd9172008-12-05 22:48:55 -08001626 tcp_write_xmit(sk, mss_now, TCP_NAGLE_PUSH, 1, sk->sk_allocation);
David S. Millerc1b4a7e2005-07-05 15:24:38 -07001627}
1628
Linus Torvalds1da177e2005-04-16 15:20:36 -07001629/* This function returns the amount that we can raise the
1630 * usable window based on the following constraints
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09001631 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07001632 * 1. The window can never be shrunk once it is offered (RFC 793)
1633 * 2. We limit memory per socket
1634 *
1635 * RFC 1122:
1636 * "the suggested [SWS] avoidance algorithm for the receiver is to keep
1637 * RECV.NEXT + RCV.WIN fixed until:
1638 * RCV.BUFF - RCV.USER - RCV.WINDOW >= min(1/2 RCV.BUFF, MSS)"
1639 *
1640 * i.e. don't raise the right edge of the window until you can raise
1641 * it at least MSS bytes.
1642 *
1643 * Unfortunately, the recommended algorithm breaks header prediction,
1644 * since header prediction assumes th->window stays fixed.
1645 *
1646 * Strictly speaking, keeping th->window fixed violates the receiver
1647 * side SWS prevention criteria. The problem is that under this rule
1648 * a stream of single byte packets will cause the right side of the
1649 * window to always advance by a single byte.
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09001650 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07001651 * Of course, if the sender implements sender side SWS prevention
1652 * then this will not be a problem.
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09001653 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07001654 * BSD seems to make the following compromise:
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09001655 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07001656 * If the free space is less than the 1/4 of the maximum
1657 * space available and the free space is less than 1/2 mss,
1658 * then set the window to 0.
1659 * [ Actually, bsd uses MSS and 1/4 of maximal _window_ ]
1660 * Otherwise, just prevent the window from shrinking
1661 * and from being larger than the largest representable value.
1662 *
1663 * This prevents incremental opening of the window in the regime
1664 * where TCP is limited by the speed of the reader side taking
1665 * data out of the TCP receive queue. It does nothing about
1666 * those cases where the window is constrained on the sender side
1667 * because the pipeline is full.
1668 *
1669 * BSD also seems to "accidentally" limit itself to windows that are a
1670 * multiple of MSS, at least until the free space gets quite small.
1671 * This would appear to be a side effect of the mbuf implementation.
1672 * Combining these two algorithms results in the observed behavior
1673 * of having a fixed window size at almost all times.
1674 *
1675 * Below we obtain similar behavior by forcing the offered window to
1676 * a multiple of the mss when it is feasible to do so.
1677 *
1678 * Note, we don't "adjust" for TIMESTAMP or SACK option bytes.
1679 * Regular options like TIMESTAMP are taken into account.
1680 */
1681u32 __tcp_select_window(struct sock *sk)
1682{
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -07001683 struct inet_connection_sock *icsk = inet_csk(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001684 struct tcp_sock *tp = tcp_sk(sk);
Stephen Hemmingercaa20d9a2005-11-10 17:13:47 -08001685 /* MSS for the peer's data. Previous versions used mss_clamp
Linus Torvalds1da177e2005-04-16 15:20:36 -07001686 * here. I don't know if the value based on our guesses
1687 * of peer's MSS is better for the performance. It's more correct
1688 * but may be worse for the performance because of rcv_mss
1689 * fluctuations. --SAW 1998/11/1
1690 */
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -07001691 int mss = icsk->icsk_ack.rcv_mss;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001692 int free_space = tcp_space(sk);
1693 int full_space = min_t(int, tp->window_clamp, tcp_full_space(sk));
1694 int window;
1695
1696 if (mss > full_space)
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09001697 mss = full_space;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001698
Eric Dumazetb92edbe2007-12-20 21:48:32 -08001699 if (free_space < (full_space >> 1)) {
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -07001700 icsk->icsk_ack.quick = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001701
1702 if (tcp_memory_pressure)
Ilpo Järvinen056834d2007-12-31 14:57:14 -08001703 tp->rcv_ssthresh = min(tp->rcv_ssthresh,
1704 4U * tp->advmss);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001705
1706 if (free_space < mss)
1707 return 0;
1708 }
1709
1710 if (free_space > tp->rcv_ssthresh)
1711 free_space = tp->rcv_ssthresh;
1712
1713 /* Don't do rounding if we are using window scaling, since the
1714 * scaled window will not line up with the MSS boundary anyway.
1715 */
1716 window = tp->rcv_wnd;
1717 if (tp->rx_opt.rcv_wscale) {
1718 window = free_space;
1719
1720 /* Advertise enough space so that it won't get scaled away.
1721 * Import case: prevent zero window announcement if
1722 * 1<<rcv_wscale > mss.
1723 */
1724 if (((window >> tp->rx_opt.rcv_wscale) << tp->rx_opt.rcv_wscale) != window)
1725 window = (((window >> tp->rx_opt.rcv_wscale) + 1)
1726 << tp->rx_opt.rcv_wscale);
1727 } else {
1728 /* Get the largest window that is a nice multiple of mss.
1729 * Window clamp already applied above.
1730 * If our current window offering is within 1 mss of the
1731 * free space we just keep it. This prevents the divide
1732 * and multiply from happening most of the time.
1733 * We also don't do any window rounding when the free space
1734 * is too small.
1735 */
1736 if (window <= free_space - mss || window > free_space)
Ilpo Järvinen056834d2007-12-31 14:57:14 -08001737 window = (free_space / mss) * mss;
John Heffner84565072007-04-02 13:56:32 -07001738 else if (mss == full_space &&
Eric Dumazetb92edbe2007-12-20 21:48:32 -08001739 free_space > window + (full_space >> 1))
John Heffner84565072007-04-02 13:56:32 -07001740 window = free_space;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001741 }
1742
1743 return window;
1744}
1745
Ilpo Järvinen4a17fc32008-11-24 21:03:43 -08001746/* Collapses two adjacent SKB's during retransmission. */
1747static void tcp_collapse_retrans(struct sock *sk, struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001748{
1749 struct tcp_sock *tp = tcp_sk(sk);
David S. Millerfe067e82007-03-07 12:12:44 -08001750 struct sk_buff *next_skb = tcp_write_queue_next(sk, skb);
Ilpo Järvinen058dc332007-12-31 04:51:11 -08001751 int skb_size, next_skb_size;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001752
Ilpo Järvinen058dc332007-12-31 04:51:11 -08001753 skb_size = skb->len;
1754 next_skb_size = next_skb->len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001755
Ilpo Järvinen058dc332007-12-31 04:51:11 -08001756 BUG_ON(tcp_skb_pcount(skb) != 1 || tcp_skb_pcount(next_skb) != 1);
Ilpo Järvinena6963a62007-09-25 22:44:14 -07001757
Ilpo Järvinen058dc332007-12-31 04:51:11 -08001758 tcp_highest_sack_combine(sk, next_skb, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001759
Ilpo Järvinen058dc332007-12-31 04:51:11 -08001760 tcp_unlink_write_queue(next_skb, sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001761
Ilpo Järvinen058dc332007-12-31 04:51:11 -08001762 skb_copy_from_linear_data(next_skb, skb_put(skb, next_skb_size),
1763 next_skb_size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001764
Ilpo Järvinen058dc332007-12-31 04:51:11 -08001765 if (next_skb->ip_summed == CHECKSUM_PARTIAL)
1766 skb->ip_summed = CHECKSUM_PARTIAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001767
Ilpo Järvinen058dc332007-12-31 04:51:11 -08001768 if (skb->ip_summed != CHECKSUM_PARTIAL)
1769 skb->csum = csum_block_add(skb->csum, next_skb->csum, skb_size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001770
Ilpo Järvinen058dc332007-12-31 04:51:11 -08001771 /* Update sequence range on original skb. */
1772 TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(next_skb)->end_seq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001773
Ilpo Järvinene6c7d082009-02-28 04:44:35 +00001774 /* Merge over control information. This moves PSH/FIN etc. over */
1775 TCP_SKB_CB(skb)->flags |= TCP_SKB_CB(next_skb)->flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001776
Ilpo Järvinen058dc332007-12-31 04:51:11 -08001777 /* All done, get rid of second SKB and account for it so
1778 * packet counting does not break.
1779 */
1780 TCP_SKB_CB(skb)->sacked |= TCP_SKB_CB(next_skb)->sacked & TCPCB_EVER_RETRANS;
Ilpo Järvinenb7689202007-09-20 11:37:19 -07001781
Ilpo Järvinen058dc332007-12-31 04:51:11 -08001782 /* changed transmit queue under us so clear hints */
Ilpo Järvinenef9da472008-09-20 21:25:15 -07001783 tcp_clear_retrans_hints_partial(tp);
1784 if (next_skb == tp->retransmit_skb_hint)
1785 tp->retransmit_skb_hint = skb;
Ilpo Järvinen058dc332007-12-31 04:51:11 -08001786
Ilpo Järvinen797108d2009-04-01 23:15:17 +00001787 tcp_adjust_pcount(sk, next_skb, tcp_skb_pcount(next_skb));
1788
Ilpo Järvinen058dc332007-12-31 04:51:11 -08001789 sk_wmem_free_skb(sk, next_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001790}
1791
Ilpo Järvinen4a17fc32008-11-24 21:03:43 -08001792static int tcp_can_collapse(struct sock *sk, struct sk_buff *skb)
1793{
1794 if (tcp_skb_pcount(skb) > 1)
1795 return 0;
1796 /* TODO: SACK collapsing could be used to remove this condition */
1797 if (skb_shinfo(skb)->nr_frags != 0)
1798 return 0;
1799 if (skb_cloned(skb))
1800 return 0;
1801 if (skb == tcp_send_head(sk))
1802 return 0;
1803 /* Some heurestics for collapsing over SACK'd could be invented */
1804 if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)
1805 return 0;
1806
1807 return 1;
1808}
1809
1810static void tcp_retrans_try_collapse(struct sock *sk, struct sk_buff *to,
1811 int space)
1812{
1813 struct tcp_sock *tp = tcp_sk(sk);
1814 struct sk_buff *skb = to, *tmp;
1815 int first = 1;
1816
1817 if (!sysctl_tcp_retrans_collapse)
1818 return;
1819 if (TCP_SKB_CB(skb)->flags & TCPCB_FLAG_SYN)
1820 return;
1821
1822 tcp_for_write_queue_from_safe(skb, tmp, sk) {
1823 if (!tcp_can_collapse(sk, skb))
1824 break;
1825
1826 space -= skb->len;
1827
1828 if (first) {
1829 first = 0;
1830 continue;
1831 }
1832
1833 if (space < 0)
1834 break;
1835 /* Punt if not enough space exists in the first SKB for
1836 * the data in the second
1837 */
1838 if (skb->len > skb_tailroom(to))
1839 break;
1840
1841 if (after(TCP_SKB_CB(skb)->end_seq, tcp_wnd_end(tp)))
1842 break;
1843
1844 tcp_collapse_retrans(sk, to);
1845 }
1846}
1847
Linus Torvalds1da177e2005-04-16 15:20:36 -07001848/* This retransmits one SKB. Policy decisions and retransmit queue
1849 * state updates are done by the caller. Returns non-zero if an
1850 * error occurred which prevented the send.
1851 */
1852int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb)
1853{
1854 struct tcp_sock *tp = tcp_sk(sk);
John Heffner5d424d52006-03-20 17:53:41 -08001855 struct inet_connection_sock *icsk = inet_csk(sk);
Sridhar Samudrala7d227cd22008-05-21 16:42:20 -07001856 unsigned int cur_mss;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001857 int err;
1858
John Heffner5d424d52006-03-20 17:53:41 -08001859 /* Inconslusive MTU probe */
1860 if (icsk->icsk_mtup.probe_size) {
1861 icsk->icsk_mtup.probe_size = 0;
1862 }
1863
Linus Torvalds1da177e2005-04-16 15:20:36 -07001864 /* Do not sent more than we queued. 1/4 is reserved for possible
Stephen Hemmingercaa20d9a2005-11-10 17:13:47 -08001865 * copying overhead: fragmentation, tunneling, mangling etc.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001866 */
1867 if (atomic_read(&sk->sk_wmem_alloc) >
1868 min(sk->sk_wmem_queued + (sk->sk_wmem_queued >> 2), sk->sk_sndbuf))
1869 return -EAGAIN;
1870
1871 if (before(TCP_SKB_CB(skb)->seq, tp->snd_una)) {
1872 if (before(TCP_SKB_CB(skb)->end_seq, tp->snd_una))
1873 BUG();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001874 if (tcp_trim_head(sk, skb, tp->snd_una - TCP_SKB_CB(skb)->seq))
1875 return -ENOMEM;
1876 }
1877
Sridhar Samudrala7d227cd22008-05-21 16:42:20 -07001878 if (inet_csk(sk)->icsk_af_ops->rebuild_header(sk))
1879 return -EHOSTUNREACH; /* Routing failure or similar. */
1880
Ilpo Järvinen0c54b852009-03-14 14:23:05 +00001881 cur_mss = tcp_current_mss(sk);
Sridhar Samudrala7d227cd22008-05-21 16:42:20 -07001882
Linus Torvalds1da177e2005-04-16 15:20:36 -07001883 /* If receiver has shrunk his window, and skb is out of
1884 * new window, do not retransmit it. The exception is the
1885 * case, when window is shrunk to zero. In this case
1886 * our retransmit serves as a zero window probe.
1887 */
Ilpo Järvinen90840de2007-12-31 04:48:41 -08001888 if (!before(TCP_SKB_CB(skb)->seq, tcp_wnd_end(tp))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001889 && TCP_SKB_CB(skb)->seq != tp->snd_una)
1890 return -EAGAIN;
1891
1892 if (skb->len > cur_mss) {
David S. Miller846998a2005-08-04 19:52:01 -07001893 if (tcp_fragment(sk, skb, cur_mss, cur_mss))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001894 return -ENOMEM; /* We'll try again later. */
Ilpo Järvinen02276f32009-02-28 04:44:31 +00001895 } else {
Ilpo Järvinen9eb93622009-04-01 23:18:20 +00001896 int oldpcount = tcp_skb_pcount(skb);
1897
1898 if (unlikely(oldpcount > 1)) {
1899 tcp_init_tso_segs(sk, skb, cur_mss);
1900 tcp_adjust_pcount(sk, skb, oldpcount - tcp_skb_pcount(skb));
1901 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001902 }
1903
Ilpo Järvinen4a17fc32008-11-24 21:03:43 -08001904 tcp_retrans_try_collapse(sk, skb, cur_mss);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001905
Linus Torvalds1da177e2005-04-16 15:20:36 -07001906 /* Some Solaris stacks overoptimize and ignore the FIN on a
1907 * retransmit when old data is attached. So strip it off
1908 * since it is cheap to do so and saves bytes on the network.
1909 */
Stephen Hemminger2de979b2007-03-08 20:45:19 -08001910 if (skb->len > 0 &&
1911 (TCP_SKB_CB(skb)->flags & TCPCB_FLAG_FIN) &&
1912 tp->snd_una == (TCP_SKB_CB(skb)->end_seq - 1)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001913 if (!pskb_trim(skb, 0)) {
Ilpo Järvinene870a8e2008-01-03 20:39:01 -08001914 /* Reuse, even though it does some unnecessary work */
1915 tcp_init_nondata_skb(skb, TCP_SKB_CB(skb)->end_seq - 1,
1916 TCP_SKB_CB(skb)->flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001917 skb->ip_summed = CHECKSUM_NONE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001918 }
1919 }
1920
1921 /* Make a copy, if the first transmission SKB clone we made
1922 * is still in somebody's hands, else make a clone.
1923 */
1924 TCP_SKB_CB(skb)->when = tcp_time_stamp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001925
David S. Millerdfb4b9d2005-12-06 16:24:52 -08001926 err = tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001927
1928 if (err == 0) {
1929 /* Update global TCP statistics. */
Pavel Emelyanov81cc8a72008-07-16 20:22:04 -07001930 TCP_INC_STATS(sock_net(sk), TCP_MIB_RETRANSSEGS);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001931
1932 tp->total_retrans++;
1933
1934#if FASTRETRANS_DEBUG > 0
Ilpo Järvinen056834d2007-12-31 14:57:14 -08001935 if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001936 if (net_ratelimit())
1937 printk(KERN_DEBUG "retrans_out leaked.\n");
1938 }
1939#endif
Ilpo Järvinenb08d6cb2007-10-11 17:36:13 -07001940 if (!tp->retrans_out)
1941 tp->lost_retrans_low = tp->snd_nxt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001942 TCP_SKB_CB(skb)->sacked |= TCPCB_RETRANS;
1943 tp->retrans_out += tcp_skb_pcount(skb);
1944
1945 /* Save stamp of the first retransmit. */
1946 if (!tp->retrans_stamp)
1947 tp->retrans_stamp = TCP_SKB_CB(skb)->when;
1948
1949 tp->undo_retrans++;
1950
1951 /* snd_nxt is stored to detect loss of retransmitted segment,
1952 * see tcp_input.c tcp_sacktag_write_queue().
1953 */
1954 TCP_SKB_CB(skb)->ack_seq = tp->snd_nxt;
1955 }
1956 return err;
1957}
1958
Ilpo Järvinenb5afe7b2008-09-20 21:21:54 -07001959static int tcp_can_forward_retransmit(struct sock *sk)
1960{
1961 const struct inet_connection_sock *icsk = inet_csk(sk);
1962 struct tcp_sock *tp = tcp_sk(sk);
1963
1964 /* Forward retransmissions are possible only during Recovery. */
1965 if (icsk->icsk_ca_state != TCP_CA_Recovery)
1966 return 0;
1967
1968 /* No forward retransmissions in Reno are possible. */
1969 if (tcp_is_reno(tp))
1970 return 0;
1971
1972 /* Yeah, we have to make difficult choice between forward transmission
1973 * and retransmission... Both ways have their merits...
1974 *
1975 * For now we do not retransmit anything, while we have some new
1976 * segments to send. In the other cases, follow rule 3 for
1977 * NextSeg() specified in RFC3517.
1978 */
1979
1980 if (tcp_may_send_now(sk))
1981 return 0;
1982
1983 return 1;
1984}
1985
Linus Torvalds1da177e2005-04-16 15:20:36 -07001986/* This gets called after a retransmit timeout, and the initially
1987 * retransmitted data is acknowledged. It tries to continue
1988 * resending the rest of the retransmit queue, until either
1989 * we've sent it all or the congestion window limit is reached.
1990 * If doing SACK, the first ACK which comes back for a timeout
1991 * based retransmit packet might feed us FACK information again.
1992 * If so, we use it to avoid unnecessarily retransmissions.
1993 */
1994void tcp_xmit_retransmit_queue(struct sock *sk)
1995{
Arnaldo Carvalho de Melo6687e982005-08-10 04:03:31 -03001996 const struct inet_connection_sock *icsk = inet_csk(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001997 struct tcp_sock *tp = tcp_sk(sk);
1998 struct sk_buff *skb;
Ilpo Järvinen0e1c54c2008-09-20 21:24:21 -07001999 struct sk_buff *hole = NULL;
Ilpo Järvinen618d9f22008-09-20 21:26:22 -07002000 u32 last_lost;
Ilpo Järvinen61eb55f2008-09-20 21:22:59 -07002001 int mib_idx;
Ilpo Järvinen0e1c54c2008-09-20 21:24:21 -07002002 int fwd_rexmitting = 0;
Stephen Hemminger6a438bb2005-11-10 17:14:59 -08002003
Ilpo Järvinen08ebd172008-09-20 21:23:49 -07002004 if (!tp->lost_out)
2005 tp->retransmit_high = tp->snd_una;
2006
Ilpo Järvinen618d9f22008-09-20 21:26:22 -07002007 if (tp->retransmit_skb_hint) {
Stephen Hemminger6a438bb2005-11-10 17:14:59 -08002008 skb = tp->retransmit_skb_hint;
Ilpo Järvinen618d9f22008-09-20 21:26:22 -07002009 last_lost = TCP_SKB_CB(skb)->end_seq;
2010 if (after(last_lost, tp->retransmit_high))
2011 last_lost = tp->retransmit_high;
2012 } else {
David S. Millerfe067e82007-03-07 12:12:44 -08002013 skb = tcp_write_queue_head(sk);
Ilpo Järvinen618d9f22008-09-20 21:26:22 -07002014 last_lost = tp->snd_una;
2015 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002016
Ilpo Järvinen08ebd172008-09-20 21:23:49 -07002017 tcp_for_write_queue_from(skb, sk) {
2018 __u8 sacked = TCP_SKB_CB(skb)->sacked;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002019
Ilpo Järvinen08ebd172008-09-20 21:23:49 -07002020 if (skb == tcp_send_head(sk))
2021 break;
2022 /* we could do better than to assign each time */
Ilpo Järvinen0e1c54c2008-09-20 21:24:21 -07002023 if (hole == NULL)
2024 tp->retransmit_skb_hint = skb;
Stephen Hemminger6a438bb2005-11-10 17:14:59 -08002025
Ilpo Järvinen08ebd172008-09-20 21:23:49 -07002026 /* Assume this retransmit will generate
2027 * only one packet for congestion window
2028 * calculation purposes. This works because
2029 * tcp_retransmit_skb() will chop up the
2030 * packet to be MSS sized and all the
2031 * packet counting works out.
2032 */
2033 if (tcp_packets_in_flight(tp) >= tp->snd_cwnd)
2034 return;
Ilpo Järvinen0e1c54c2008-09-20 21:24:21 -07002035
2036 if (fwd_rexmitting) {
2037begin_fwd:
2038 if (!before(TCP_SKB_CB(skb)->seq, tcp_highest_sack_seq(tp)))
2039 break;
2040 mib_idx = LINUX_MIB_TCPFORWARDRETRANS;
2041
2042 } else if (!before(TCP_SKB_CB(skb)->seq, tp->retransmit_high)) {
Ilpo Järvinen618d9f22008-09-20 21:26:22 -07002043 tp->retransmit_high = last_lost;
Ilpo Järvinen0e1c54c2008-09-20 21:24:21 -07002044 if (!tcp_can_forward_retransmit(sk))
2045 break;
2046 /* Backtrack if necessary to non-L'ed skb */
2047 if (hole != NULL) {
2048 skb = hole;
2049 hole = NULL;
2050 }
2051 fwd_rexmitting = 1;
2052 goto begin_fwd;
2053
2054 } else if (!(sacked & TCPCB_LOST)) {
Ilpo Järvinenac11ba72009-02-28 04:44:27 +00002055 if (hole == NULL && !(sacked & (TCPCB_SACKED_RETRANS|TCPCB_SACKED_ACKED)))
Ilpo Järvinen0e1c54c2008-09-20 21:24:21 -07002056 hole = skb;
Ilpo Järvinen08ebd172008-09-20 21:23:49 -07002057 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002058
Ilpo Järvinen0e1c54c2008-09-20 21:24:21 -07002059 } else {
Ilpo Järvinen618d9f22008-09-20 21:26:22 -07002060 last_lost = TCP_SKB_CB(skb)->end_seq;
Ilpo Järvinen0e1c54c2008-09-20 21:24:21 -07002061 if (icsk->icsk_ca_state != TCP_CA_Loss)
2062 mib_idx = LINUX_MIB_TCPFASTRETRANS;
2063 else
2064 mib_idx = LINUX_MIB_TCPSLOWSTARTRETRANS;
2065 }
2066
2067 if (sacked & (TCPCB_SACKED_ACKED|TCPCB_SACKED_RETRANS))
Ilpo Järvinen08ebd172008-09-20 21:23:49 -07002068 continue;
Pavel Emelyanov40b215e2008-07-03 01:05:41 -07002069
Ilpo Järvinenf0ceb0e2008-09-20 21:24:49 -07002070 if (tcp_retransmit_skb(sk, skb))
Ilpo Järvinen08ebd172008-09-20 21:23:49 -07002071 return;
Ilpo Järvinen08ebd172008-09-20 21:23:49 -07002072 NET_INC_STATS_BH(sock_net(sk), mib_idx);
2073
2074 if (skb == tcp_write_queue_head(sk))
2075 inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
2076 inet_csk(sk)->icsk_rto,
2077 TCP_RTO_MAX);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002078 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002079}
2080
Linus Torvalds1da177e2005-04-16 15:20:36 -07002081/* Send a fin. The caller locks the socket for us. This cannot be
2082 * allowed to fail queueing a FIN frame under any circumstances.
2083 */
2084void tcp_send_fin(struct sock *sk)
2085{
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09002086 struct tcp_sock *tp = tcp_sk(sk);
David S. Millerfe067e82007-03-07 12:12:44 -08002087 struct sk_buff *skb = tcp_write_queue_tail(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002088 int mss_now;
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09002089
Linus Torvalds1da177e2005-04-16 15:20:36 -07002090 /* Optimization, tack on the FIN if we have a queue of
2091 * unsent frames. But be careful about outgoing SACKS
2092 * and IP options.
2093 */
Ilpo Järvinen0c54b852009-03-14 14:23:05 +00002094 mss_now = tcp_current_mss(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002095
David S. Millerfe067e82007-03-07 12:12:44 -08002096 if (tcp_send_head(sk) != NULL) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002097 TCP_SKB_CB(skb)->flags |= TCPCB_FLAG_FIN;
2098 TCP_SKB_CB(skb)->end_seq++;
2099 tp->write_seq++;
2100 } else {
2101 /* Socket is locked, keep trying until memory is available. */
2102 for (;;) {
David S. Millerd179cd12005-08-17 14:57:30 -07002103 skb = alloc_skb_fclone(MAX_TCP_HEADER, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002104 if (skb)
2105 break;
2106 yield();
2107 }
2108
2109 /* Reserve space for headers and prepare control bits. */
2110 skb_reserve(skb, MAX_TCP_HEADER);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002111 /* FIN eats a sequence byte, write_seq advanced by tcp_queue_skb(). */
Ilpo Järvinene870a8e2008-01-03 20:39:01 -08002112 tcp_init_nondata_skb(skb, tp->write_seq,
2113 TCPCB_FLAG_ACK | TCPCB_FLAG_FIN);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002114 tcp_queue_skb(sk, skb);
2115 }
Ilpo Järvinen9e412ba2007-04-20 22:18:02 -07002116 __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_OFF);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002117}
2118
2119/* We get here when a process closes a file descriptor (either due to
2120 * an explicit close() or as a byproduct of exit()'ing) and there
2121 * was unread data in the receive queue. This behavior is recommended
Gerrit Renker65bb7232007-04-28 21:21:46 -07002122 * by RFC 2525, section 2.17. -DaveM
Linus Torvalds1da177e2005-04-16 15:20:36 -07002123 */
Al Virodd0fc662005-10-07 07:46:04 +01002124void tcp_send_active_reset(struct sock *sk, gfp_t priority)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002125{
Linus Torvalds1da177e2005-04-16 15:20:36 -07002126 struct sk_buff *skb;
2127
2128 /* NOTE: No TCP options attached and we never retransmit this. */
2129 skb = alloc_skb(MAX_TCP_HEADER, priority);
2130 if (!skb) {
Pavel Emelyanov4e673442008-07-16 20:30:14 -07002131 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTFAILED);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002132 return;
2133 }
2134
2135 /* Reserve space for headers and prepare control bits. */
2136 skb_reserve(skb, MAX_TCP_HEADER);
Ilpo Järvinene870a8e2008-01-03 20:39:01 -08002137 tcp_init_nondata_skb(skb, tcp_acceptable_seq(sk),
2138 TCPCB_FLAG_ACK | TCPCB_FLAG_RST);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002139 /* Send it off. */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002140 TCP_SKB_CB(skb)->when = tcp_time_stamp;
David S. Millerdfb4b9d2005-12-06 16:24:52 -08002141 if (tcp_transmit_skb(sk, skb, 0, priority))
Pavel Emelyanov4e673442008-07-16 20:30:14 -07002142 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTFAILED);
Sridhar Samudrala26af65c2008-06-04 15:19:35 -07002143
Pavel Emelyanov81cc8a72008-07-16 20:22:04 -07002144 TCP_INC_STATS(sock_net(sk), TCP_MIB_OUTRSTS);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002145}
2146
2147/* WARNING: This routine must only be called when we have already sent
2148 * a SYN packet that crossed the incoming SYN that caused this routine
2149 * to get called. If this assumption fails then the initial rcv_wnd
2150 * and rcv_wscale values will not be correct.
2151 */
2152int tcp_send_synack(struct sock *sk)
2153{
Ilpo Järvinen056834d2007-12-31 14:57:14 -08002154 struct sk_buff *skb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002155
David S. Millerfe067e82007-03-07 12:12:44 -08002156 skb = tcp_write_queue_head(sk);
Ilpo Järvinen056834d2007-12-31 14:57:14 -08002157 if (skb == NULL || !(TCP_SKB_CB(skb)->flags & TCPCB_FLAG_SYN)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002158 printk(KERN_DEBUG "tcp_send_synack: wrong queue state\n");
2159 return -EFAULT;
2160 }
Ilpo Järvinen056834d2007-12-31 14:57:14 -08002161 if (!(TCP_SKB_CB(skb)->flags & TCPCB_FLAG_ACK)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002162 if (skb_cloned(skb)) {
2163 struct sk_buff *nskb = skb_copy(skb, GFP_ATOMIC);
2164 if (nskb == NULL)
2165 return -ENOMEM;
David S. Millerfe067e82007-03-07 12:12:44 -08002166 tcp_unlink_write_queue(skb, sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002167 skb_header_release(nskb);
David S. Millerfe067e82007-03-07 12:12:44 -08002168 __tcp_add_write_queue_head(sk, nskb);
Hideo Aoki3ab224b2007-12-31 00:11:19 -08002169 sk_wmem_free_skb(sk, skb);
2170 sk->sk_wmem_queued += nskb->truesize;
2171 sk_mem_charge(sk, nskb->truesize);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002172 skb = nskb;
2173 }
2174
2175 TCP_SKB_CB(skb)->flags |= TCPCB_FLAG_ACK;
2176 TCP_ECN_send_synack(tcp_sk(sk), skb);
2177 }
2178 TCP_SKB_CB(skb)->when = tcp_time_stamp;
David S. Millerdfb4b9d2005-12-06 16:24:52 -08002179 return tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002180}
2181
2182/*
2183 * Prepare a SYN-ACK.
2184 */
Ilpo Järvinen056834d2007-12-31 14:57:14 -08002185struct sk_buff *tcp_make_synack(struct sock *sk, struct dst_entry *dst,
2186 struct request_sock *req)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002187{
Arnaldo Carvalho de Melo2e6599c2005-06-18 22:46:52 -07002188 struct inet_request_sock *ireq = inet_rsk(req);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002189 struct tcp_sock *tp = tcp_sk(sk);
2190 struct tcphdr *th;
2191 int tcp_header_size;
Adam Langley33ad7982008-07-19 00:04:31 -07002192 struct tcp_out_options opts;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002193 struct sk_buff *skb;
YOSHIFUJI Hideakicfb6eeb2006-11-14 19:07:45 -08002194 struct tcp_md5sig_key *md5;
2195 __u8 *md5_hash_location;
Tom Quetchenbachf5fff5d2008-09-21 00:21:51 -07002196 int mss;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002197
2198 skb = sock_wmalloc(sk, MAX_TCP_HEADER + 15, 1, GFP_ATOMIC);
2199 if (skb == NULL)
2200 return NULL;
2201
2202 /* Reserve space for headers. */
2203 skb_reserve(skb, MAX_TCP_HEADER);
2204
2205 skb->dst = dst_clone(dst);
2206
Tom Quetchenbachf5fff5d2008-09-21 00:21:51 -07002207 mss = dst_metric(dst, RTAX_ADVMSS);
2208 if (tp->rx_opt.user_mss && tp->rx_opt.user_mss < mss)
2209 mss = tp->rx_opt.user_mss;
2210
Adam Langley33ad7982008-07-19 00:04:31 -07002211 if (req->rcv_wnd == 0) { /* ignored for retransmitted syns */
2212 __u8 rcv_wscale;
2213 /* Set this up on the first call only */
2214 req->window_clamp = tp->window_clamp ? : dst_metric(dst, RTAX_WINDOW);
2215 /* tcp_full_space because it is guaranteed to be the first packet */
2216 tcp_select_initial_window(tcp_full_space(sk),
Tom Quetchenbachf5fff5d2008-09-21 00:21:51 -07002217 mss - (ireq->tstamp_ok ? TCPOLEN_TSTAMP_ALIGNED : 0),
Adam Langley33ad7982008-07-19 00:04:31 -07002218 &req->rcv_wnd,
2219 &req->window_clamp,
2220 ireq->wscale_ok,
2221 &rcv_wscale);
2222 ireq->rcv_wscale = rcv_wscale;
2223 }
YOSHIFUJI Hideakicfb6eeb2006-11-14 19:07:45 -08002224
Adam Langley33ad7982008-07-19 00:04:31 -07002225 memset(&opts, 0, sizeof(opts));
Florian Westphal8b5f12d2008-10-26 23:10:12 -07002226#ifdef CONFIG_SYN_COOKIES
2227 if (unlikely(req->cookie_ts))
2228 TCP_SKB_CB(skb)->when = cookie_init_timestamp(req);
2229 else
2230#endif
Adam Langley33ad7982008-07-19 00:04:31 -07002231 TCP_SKB_CB(skb)->when = tcp_time_stamp;
Tom Quetchenbachf5fff5d2008-09-21 00:21:51 -07002232 tcp_header_size = tcp_synack_options(sk, req, mss,
Adam Langley33ad7982008-07-19 00:04:31 -07002233 skb, &opts, &md5) +
2234 sizeof(struct tcphdr);
2235
Arnaldo Carvalho de Meloaa8223c2007-04-10 21:04:22 -07002236 skb_push(skb, tcp_header_size);
2237 skb_reset_transport_header(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002238
Arnaldo Carvalho de Meloaa8223c2007-04-10 21:04:22 -07002239 th = tcp_hdr(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002240 memset(th, 0, sizeof(struct tcphdr));
2241 th->syn = 1;
2242 th->ack = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002243 TCP_ECN_make_synack(req, th);
KOVACS Krisztiana3116ac52008-10-01 07:46:49 -07002244 th->source = ireq->loc_port;
Arnaldo Carvalho de Melo2e6599c2005-06-18 22:46:52 -07002245 th->dest = ireq->rmt_port;
Ilpo Järvinene870a8e2008-01-03 20:39:01 -08002246 /* Setting of flags are superfluous here for callers (and ECE is
2247 * not even correctly set)
2248 */
2249 tcp_init_nondata_skb(skb, tcp_rsk(req)->snt_isn,
2250 TCPCB_FLAG_SYN | TCPCB_FLAG_ACK);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002251 th->seq = htonl(TCP_SKB_CB(skb)->seq);
Arnaldo Carvalho de Melo2e6599c2005-06-18 22:46:52 -07002252 th->ack_seq = htonl(tcp_rsk(req)->rcv_isn + 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002253
2254 /* RFC1323: The window in SYN & SYN/ACK segments is never scaled. */
Ilpo Järvinen600ff0c2007-02-13 12:42:11 -08002255 th->window = htons(min(req->rcv_wnd, 65535U));
Adam Langley33ad7982008-07-19 00:04:31 -07002256 tcp_options_write((__be32 *)(th + 1), tp, &opts, &md5_hash_location);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002257 th->doff = (tcp_header_size >> 2);
Pavel Emelyanov81cc8a72008-07-16 20:22:04 -07002258 TCP_INC_STATS(sock_net(sk), TCP_MIB_OUTSEGS);
YOSHIFUJI Hideakicfb6eeb2006-11-14 19:07:45 -08002259
2260#ifdef CONFIG_TCP_MD5SIG
2261 /* Okay, we have all we need - do the md5 hash if needed */
2262 if (md5) {
2263 tp->af_specific->calc_md5_hash(md5_hash_location,
Adam Langley49a72df2008-07-19 00:01:42 -07002264 md5, NULL, req, skb);
YOSHIFUJI Hideakicfb6eeb2006-11-14 19:07:45 -08002265 }
2266#endif
2267
Linus Torvalds1da177e2005-04-16 15:20:36 -07002268 return skb;
2269}
2270
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09002271/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07002272 * Do all connect socket setups that can be done AF independent.
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09002273 */
Stephen Hemminger40efc6f2006-01-03 16:03:49 -08002274static void tcp_connect_init(struct sock *sk)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002275{
2276 struct dst_entry *dst = __sk_dst_get(sk);
2277 struct tcp_sock *tp = tcp_sk(sk);
2278 __u8 rcv_wscale;
2279
2280 /* We'll fix this up when we get a response from the other end.
2281 * See tcp_input.c:tcp_rcv_state_process case TCP_SYN_SENT.
2282 */
2283 tp->tcp_header_len = sizeof(struct tcphdr) +
2284 (sysctl_tcp_timestamps ? TCPOLEN_TSTAMP_ALIGNED : 0);
2285
YOSHIFUJI Hideakicfb6eeb2006-11-14 19:07:45 -08002286#ifdef CONFIG_TCP_MD5SIG
2287 if (tp->af_specific->md5_lookup(sk, sk) != NULL)
2288 tp->tcp_header_len += TCPOLEN_MD5SIG_ALIGNED;
2289#endif
2290
Linus Torvalds1da177e2005-04-16 15:20:36 -07002291 /* If user gave his TCP_MAXSEG, record it to clamp */
2292 if (tp->rx_opt.user_mss)
2293 tp->rx_opt.mss_clamp = tp->rx_opt.user_mss;
2294 tp->max_window = 0;
John Heffner5d424d52006-03-20 17:53:41 -08002295 tcp_mtup_init(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002296 tcp_sync_mss(sk, dst_mtu(dst));
2297
2298 if (!tp->window_clamp)
2299 tp->window_clamp = dst_metric(dst, RTAX_WINDOW);
2300 tp->advmss = dst_metric(dst, RTAX_ADVMSS);
Tom Quetchenbachf5fff5d2008-09-21 00:21:51 -07002301 if (tp->rx_opt.user_mss && tp->rx_opt.user_mss < tp->advmss)
2302 tp->advmss = tp->rx_opt.user_mss;
2303
Linus Torvalds1da177e2005-04-16 15:20:36 -07002304 tcp_initialize_rcv_mss(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002305
2306 tcp_select_initial_window(tcp_full_space(sk),
2307 tp->advmss - (tp->rx_opt.ts_recent_stamp ? tp->tcp_header_len - sizeof(struct tcphdr) : 0),
2308 &tp->rcv_wnd,
2309 &tp->window_clamp,
2310 sysctl_tcp_window_scaling,
2311 &rcv_wscale);
2312
2313 tp->rx_opt.rcv_wscale = rcv_wscale;
2314 tp->rcv_ssthresh = tp->rcv_wnd;
2315
2316 sk->sk_err = 0;
2317 sock_reset_flag(sk, SOCK_DONE);
2318 tp->snd_wnd = 0;
Hantzis Fotisee7537b2009-03-02 22:42:02 -08002319 tcp_init_wl(tp, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002320 tp->snd_una = tp->write_seq;
2321 tp->snd_sml = tp->write_seq;
Ilpo Järvinen33f5f572008-10-07 14:43:06 -07002322 tp->snd_up = tp->write_seq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002323 tp->rcv_nxt = 0;
2324 tp->rcv_wup = 0;
2325 tp->copied_seq = 0;
2326
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -07002327 inet_csk(sk)->icsk_rto = TCP_TIMEOUT_INIT;
2328 inet_csk(sk)->icsk_retransmits = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002329 tcp_clear_retrans(tp);
2330}
2331
2332/*
2333 * Build a SYN and send it off.
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09002334 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002335int tcp_connect(struct sock *sk)
2336{
2337 struct tcp_sock *tp = tcp_sk(sk);
2338 struct sk_buff *buff;
2339
2340 tcp_connect_init(sk);
2341
David S. Millerd179cd12005-08-17 14:57:30 -07002342 buff = alloc_skb_fclone(MAX_TCP_HEADER + 15, sk->sk_allocation);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002343 if (unlikely(buff == NULL))
2344 return -ENOBUFS;
2345
2346 /* Reserve space for headers. */
2347 skb_reserve(buff, MAX_TCP_HEADER);
2348
Wei Yongjunbd37a082006-08-07 21:04:15 -07002349 tp->snd_nxt = tp->write_seq;
Ilpo Järvinene870a8e2008-01-03 20:39:01 -08002350 tcp_init_nondata_skb(buff, tp->write_seq++, TCPCB_FLAG_SYN);
2351 TCP_ECN_send_syn(sk, buff);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002352
2353 /* Send it off. */
2354 TCP_SKB_CB(buff)->when = tcp_time_stamp;
2355 tp->retrans_stamp = TCP_SKB_CB(buff)->when;
2356 skb_header_release(buff);
David S. Millerfe067e82007-03-07 12:12:44 -08002357 __tcp_add_write_queue_tail(sk, buff);
Hideo Aoki3ab224b2007-12-31 00:11:19 -08002358 sk->sk_wmem_queued += buff->truesize;
2359 sk_mem_charge(sk, buff->truesize);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002360 tp->packets_out += tcp_skb_pcount(buff);
David S. Millerdfb4b9d2005-12-06 16:24:52 -08002361 tcp_transmit_skb(sk, buff, 1, GFP_KERNEL);
Wei Yongjunbd37a082006-08-07 21:04:15 -07002362
2363 /* We change tp->snd_nxt after the tcp_transmit_skb() call
2364 * in order to make this packet get counted in tcpOutSegs.
2365 */
2366 tp->snd_nxt = tp->write_seq;
2367 tp->pushed_seq = tp->write_seq;
Pavel Emelyanov81cc8a72008-07-16 20:22:04 -07002368 TCP_INC_STATS(sock_net(sk), TCP_MIB_ACTIVEOPENS);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002369
2370 /* Timer for repeating the SYN until an answer. */
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -07002371 inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
2372 inet_csk(sk)->icsk_rto, TCP_RTO_MAX);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002373 return 0;
2374}
2375
2376/* Send out a delayed ack, the caller does the policy checking
2377 * to see if we should even be here. See tcp_input.c:tcp_ack_snd_check()
2378 * for details.
2379 */
2380void tcp_send_delayed_ack(struct sock *sk)
2381{
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -07002382 struct inet_connection_sock *icsk = inet_csk(sk);
2383 int ato = icsk->icsk_ack.ato;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002384 unsigned long timeout;
2385
2386 if (ato > TCP_DELACK_MIN) {
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -07002387 const struct tcp_sock *tp = tcp_sk(sk);
Ilpo Järvinen056834d2007-12-31 14:57:14 -08002388 int max_ato = HZ / 2;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002389
Ilpo Järvinen056834d2007-12-31 14:57:14 -08002390 if (icsk->icsk_ack.pingpong ||
2391 (icsk->icsk_ack.pending & ICSK_ACK_PUSHED))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002392 max_ato = TCP_DELACK_MAX;
2393
2394 /* Slow path, intersegment interval is "high". */
2395
2396 /* If some rtt estimate is known, use it to bound delayed ack.
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -07002397 * Do not use inet_csk(sk)->icsk_rto here, use results of rtt measurements
Linus Torvalds1da177e2005-04-16 15:20:36 -07002398 * directly.
2399 */
2400 if (tp->srtt) {
Ilpo Järvinen056834d2007-12-31 14:57:14 -08002401 int rtt = max(tp->srtt >> 3, TCP_DELACK_MIN);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002402
2403 if (rtt < max_ato)
2404 max_ato = rtt;
2405 }
2406
2407 ato = min(ato, max_ato);
2408 }
2409
2410 /* Stay within the limit we were given */
2411 timeout = jiffies + ato;
2412
2413 /* Use new timeout only if there wasn't a older one earlier. */
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -07002414 if (icsk->icsk_ack.pending & ICSK_ACK_TIMER) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002415 /* If delack timer was blocked or is about to expire,
2416 * send ACK now.
2417 */
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -07002418 if (icsk->icsk_ack.blocked ||
2419 time_before_eq(icsk->icsk_ack.timeout, jiffies + (ato >> 2))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002420 tcp_send_ack(sk);
2421 return;
2422 }
2423
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -07002424 if (!time_before(timeout, icsk->icsk_ack.timeout))
2425 timeout = icsk->icsk_ack.timeout;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002426 }
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -07002427 icsk->icsk_ack.pending |= ICSK_ACK_SCHED | ICSK_ACK_TIMER;
2428 icsk->icsk_ack.timeout = timeout;
2429 sk_reset_timer(sk, &icsk->icsk_delack_timer, timeout);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002430}
2431
2432/* This routine sends an ack and also updates the window. */
2433void tcp_send_ack(struct sock *sk)
2434{
Ilpo Järvinen058dc332007-12-31 04:51:11 -08002435 struct sk_buff *buff;
2436
Linus Torvalds1da177e2005-04-16 15:20:36 -07002437 /* If we have been reset, we may not send again. */
Ilpo Järvinen058dc332007-12-31 04:51:11 -08002438 if (sk->sk_state == TCP_CLOSE)
2439 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002440
Ilpo Järvinen058dc332007-12-31 04:51:11 -08002441 /* We are not putting this on the write queue, so
2442 * tcp_transmit_skb() will set the ownership to this
2443 * sock.
2444 */
2445 buff = alloc_skb(MAX_TCP_HEADER, GFP_ATOMIC);
2446 if (buff == NULL) {
2447 inet_csk_schedule_ack(sk);
2448 inet_csk(sk)->icsk_ack.ato = TCP_ATO_MIN;
2449 inet_csk_reset_xmit_timer(sk, ICSK_TIME_DACK,
2450 TCP_DELACK_MAX, TCP_RTO_MAX);
2451 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002452 }
Ilpo Järvinen058dc332007-12-31 04:51:11 -08002453
2454 /* Reserve space for headers and prepare control bits. */
2455 skb_reserve(buff, MAX_TCP_HEADER);
Ilpo Järvinene870a8e2008-01-03 20:39:01 -08002456 tcp_init_nondata_skb(buff, tcp_acceptable_seq(sk), TCPCB_FLAG_ACK);
Ilpo Järvinen058dc332007-12-31 04:51:11 -08002457
2458 /* Send it off, this clears delayed acks for us. */
Ilpo Järvinen058dc332007-12-31 04:51:11 -08002459 TCP_SKB_CB(buff)->when = tcp_time_stamp;
2460 tcp_transmit_skb(sk, buff, 0, GFP_ATOMIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002461}
2462
2463/* This routine sends a packet with an out of date sequence
2464 * number. It assumes the other end will try to ack it.
2465 *
2466 * Question: what should we make while urgent mode?
2467 * 4.4BSD forces sending single byte of data. We cannot send
2468 * out of window data, because we have SND.NXT==SND.MAX...
2469 *
2470 * Current solution: to send TWO zero-length segments in urgent mode:
2471 * one is with SEG.SEQ=SND.UNA to deliver urgent pointer, another is
2472 * out-of-date with SND.UNA-1 to probe window.
2473 */
2474static int tcp_xmit_probe_skb(struct sock *sk, int urgent)
2475{
2476 struct tcp_sock *tp = tcp_sk(sk);
2477 struct sk_buff *skb;
2478
2479 /* We don't queue it, tcp_transmit_skb() sets ownership. */
2480 skb = alloc_skb(MAX_TCP_HEADER, GFP_ATOMIC);
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09002481 if (skb == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002482 return -1;
2483
2484 /* Reserve space for headers and set control bits. */
2485 skb_reserve(skb, MAX_TCP_HEADER);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002486 /* Use a previous sequence. This should cause the other
2487 * end to send an ack. Don't queue or clone SKB, just
2488 * send it.
2489 */
Ilpo Järvinene870a8e2008-01-03 20:39:01 -08002490 tcp_init_nondata_skb(skb, tp->snd_una - !urgent, TCPCB_FLAG_ACK);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002491 TCP_SKB_CB(skb)->when = tcp_time_stamp;
David S. Millerdfb4b9d2005-12-06 16:24:52 -08002492 return tcp_transmit_skb(sk, skb, 0, GFP_ATOMIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002493}
2494
2495int tcp_write_wakeup(struct sock *sk)
2496{
Ilpo Järvinen058dc332007-12-31 04:51:11 -08002497 struct tcp_sock *tp = tcp_sk(sk);
2498 struct sk_buff *skb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002499
Ilpo Järvinen058dc332007-12-31 04:51:11 -08002500 if (sk->sk_state == TCP_CLOSE)
2501 return -1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002502
Ilpo Järvinen058dc332007-12-31 04:51:11 -08002503 if ((skb = tcp_send_head(sk)) != NULL &&
2504 before(TCP_SKB_CB(skb)->seq, tcp_wnd_end(tp))) {
2505 int err;
Ilpo Järvinen0c54b852009-03-14 14:23:05 +00002506 unsigned int mss = tcp_current_mss(sk);
Ilpo Järvinen058dc332007-12-31 04:51:11 -08002507 unsigned int seg_size = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002508
Ilpo Järvinen058dc332007-12-31 04:51:11 -08002509 if (before(tp->pushed_seq, TCP_SKB_CB(skb)->end_seq))
2510 tp->pushed_seq = TCP_SKB_CB(skb)->end_seq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002511
Ilpo Järvinen058dc332007-12-31 04:51:11 -08002512 /* We are probing the opening of a window
2513 * but the window size is != 0
2514 * must have been a result SWS avoidance ( sender )
2515 */
2516 if (seg_size < TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq ||
2517 skb->len > mss) {
2518 seg_size = min(seg_size, mss);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002519 TCP_SKB_CB(skb)->flags |= TCPCB_FLAG_PSH;
Ilpo Järvinen058dc332007-12-31 04:51:11 -08002520 if (tcp_fragment(sk, skb, seg_size, mss))
2521 return -1;
2522 } else if (!tcp_skb_pcount(skb))
2523 tcp_set_skb_tso_segs(sk, skb, mss);
2524
2525 TCP_SKB_CB(skb)->flags |= TCPCB_FLAG_PSH;
2526 TCP_SKB_CB(skb)->when = tcp_time_stamp;
2527 err = tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC);
2528 if (!err)
2529 tcp_event_new_data_sent(sk, skb);
2530 return err;
2531 } else {
Ilpo Järvinen33f5f572008-10-07 14:43:06 -07002532 if (between(tp->snd_up, tp->snd_una + 1, tp->snd_una + 0xFFFF))
Ilpo Järvinen058dc332007-12-31 04:51:11 -08002533 tcp_xmit_probe_skb(sk, 1);
2534 return tcp_xmit_probe_skb(sk, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002535 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002536}
2537
2538/* A window probe timeout has occurred. If window is not closed send
2539 * a partial packet else a zero probe.
2540 */
2541void tcp_send_probe0(struct sock *sk)
2542{
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -07002543 struct inet_connection_sock *icsk = inet_csk(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002544 struct tcp_sock *tp = tcp_sk(sk);
2545 int err;
2546
2547 err = tcp_write_wakeup(sk);
2548
David S. Millerfe067e82007-03-07 12:12:44 -08002549 if (tp->packets_out || !tcp_send_head(sk)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002550 /* Cancel probe timer, if it is not required. */
Arnaldo Carvalho de Melo6687e982005-08-10 04:03:31 -03002551 icsk->icsk_probes_out = 0;
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -07002552 icsk->icsk_backoff = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002553 return;
2554 }
2555
2556 if (err <= 0) {
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -07002557 if (icsk->icsk_backoff < sysctl_tcp_retries2)
2558 icsk->icsk_backoff++;
Arnaldo Carvalho de Melo6687e982005-08-10 04:03:31 -03002559 icsk->icsk_probes_out++;
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09002560 inet_csk_reset_xmit_timer(sk, ICSK_TIME_PROBE0,
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -07002561 min(icsk->icsk_rto << icsk->icsk_backoff, TCP_RTO_MAX),
2562 TCP_RTO_MAX);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002563 } else {
2564 /* If packet was not sent due to local congestion,
Arnaldo Carvalho de Melo6687e982005-08-10 04:03:31 -03002565 * do not backoff and do not remember icsk_probes_out.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002566 * Let local senders to fight for local resources.
2567 *
2568 * Use accumulated backoff yet.
2569 */
Arnaldo Carvalho de Melo6687e982005-08-10 04:03:31 -03002570 if (!icsk->icsk_probes_out)
2571 icsk->icsk_probes_out = 1;
YOSHIFUJI Hideakie905a9e2007-02-09 23:24:47 +09002572 inet_csk_reset_xmit_timer(sk, ICSK_TIME_PROBE0,
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -07002573 min(icsk->icsk_rto << icsk->icsk_backoff,
Arnaldo Carvalho de Melo3f421ba2005-08-09 20:11:08 -07002574 TCP_RESOURCE_PROBE_INTERVAL),
2575 TCP_RTO_MAX);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002576 }
2577}
2578
Glenn Griffinc6aefaf2008-02-07 21:49:26 -08002579EXPORT_SYMBOL(tcp_select_initial_window);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002580EXPORT_SYMBOL(tcp_connect);
2581EXPORT_SYMBOL(tcp_make_synack);
2582EXPORT_SYMBOL(tcp_simple_retransmit);
2583EXPORT_SYMBOL(tcp_sync_mss);
John Heffner5d424d52006-03-20 17:53:41 -08002584EXPORT_SYMBOL(tcp_mtup_init);