blob: 06e72d3cdf60dbd96a3ad4829cfcc2baec0a80e8 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Routines having to do with the 'struct sk_buff' memory handlers.
3 *
Alan Cox113aa832008-10-13 19:01:08 -07004 * Authors: Alan Cox <alan@lxorguk.ukuu.org.uk>
Linus Torvalds1da177e2005-04-16 15:20:36 -07005 * Florian La Roche <rzsfl@rz.uni-sb.de>
6 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07007 * Fixes:
8 * Alan Cox : Fixed the worst of the load
9 * balancer bugs.
10 * Dave Platt : Interrupt stacking fix.
11 * Richard Kooijman : Timestamp fixes.
12 * Alan Cox : Changed buffer format.
13 * Alan Cox : destructor hook for AF_UNIX etc.
14 * Linus Torvalds : Better skb_clone.
15 * Alan Cox : Added skb_copy.
16 * Alan Cox : Added all the changed routines Linus
17 * only put in the headers
18 * Ray VanTassle : Fixed --skb->lock in free
19 * Alan Cox : skb_copy copy arp field
20 * Andi Kleen : slabified it.
21 * Robert Olsson : Removed skb_head_pool
22 *
23 * NOTE:
24 * The __skb_ routines should be called with interrupts
25 * disabled, or you better be *real* sure that the operation is atomic
26 * with respect to whatever list is being frobbed (e.g. via lock_sock()
27 * or via disabling bottom half handlers, etc).
28 *
29 * This program is free software; you can redistribute it and/or
30 * modify it under the terms of the GNU General Public License
31 * as published by the Free Software Foundation; either version
32 * 2 of the License, or (at your option) any later version.
33 */
34
35/*
36 * The functions in this file will not compile correctly with gcc 2.4.x
37 */
38
Joe Perchese005d192012-05-16 19:58:40 +000039#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
40
Linus Torvalds1da177e2005-04-16 15:20:36 -070041#include <linux/module.h>
42#include <linux/types.h>
43#include <linux/kernel.h>
Vegard Nossumfe55f6d2008-08-30 12:16:35 +020044#include <linux/kmemcheck.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070045#include <linux/mm.h>
46#include <linux/interrupt.h>
47#include <linux/in.h>
48#include <linux/inet.h>
49#include <linux/slab.h>
50#include <linux/netdevice.h>
51#ifdef CONFIG_NET_CLS_ACT
52#include <net/pkt_sched.h>
53#endif
54#include <linux/string.h>
55#include <linux/skbuff.h>
Jens Axboe9c55e012007-11-06 23:30:13 -080056#include <linux/splice.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070057#include <linux/cache.h>
58#include <linux/rtnetlink.h>
59#include <linux/init.h>
David Howells716ea3a2007-04-02 20:19:53 -070060#include <linux/scatterlist.h>
Patrick Ohlyac45f602009-02-12 05:03:37 +000061#include <linux/errqueue.h>
Linus Torvalds268bb0c2011-05-20 12:50:29 -070062#include <linux/prefetch.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070063
64#include <net/protocol.h>
65#include <net/dst.h>
66#include <net/sock.h>
67#include <net/checksum.h>
68#include <net/xfrm.h>
69
70#include <asm/uaccess.h>
Steven Rostedtad8d75f2009-04-14 19:39:12 -040071#include <trace/events/skb.h>
Eric Dumazet51c56b02012-04-05 11:35:15 +020072#include <linux/highmem.h>
Al Viroa1f8e7f72006-10-19 16:08:53 -040073
Eric Dumazetd7e88832012-04-30 08:10:34 +000074struct kmem_cache *skbuff_head_cache __read_mostly;
Christoph Lametere18b8902006-12-06 20:33:20 -080075static struct kmem_cache *skbuff_fclone_cache __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -070076
Jens Axboe9c55e012007-11-06 23:30:13 -080077static void sock_pipe_buf_release(struct pipe_inode_info *pipe,
78 struct pipe_buffer *buf)
79{
Jarek Poplawski8b9d3722009-01-19 17:03:56 -080080 put_page(buf->page);
Jens Axboe9c55e012007-11-06 23:30:13 -080081}
82
83static void sock_pipe_buf_get(struct pipe_inode_info *pipe,
84 struct pipe_buffer *buf)
85{
Jarek Poplawski8b9d3722009-01-19 17:03:56 -080086 get_page(buf->page);
Jens Axboe9c55e012007-11-06 23:30:13 -080087}
88
89static int sock_pipe_buf_steal(struct pipe_inode_info *pipe,
90 struct pipe_buffer *buf)
91{
92 return 1;
93}
94
95
96/* Pipe buffer operations for a socket. */
Alexey Dobriyan28dfef82009-12-15 16:46:48 -080097static const struct pipe_buf_operations sock_pipe_buf_ops = {
Jens Axboe9c55e012007-11-06 23:30:13 -080098 .can_merge = 0,
99 .map = generic_pipe_buf_map,
100 .unmap = generic_pipe_buf_unmap,
101 .confirm = generic_pipe_buf_confirm,
102 .release = sock_pipe_buf_release,
103 .steal = sock_pipe_buf_steal,
104 .get = sock_pipe_buf_get,
105};
106
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107/**
Jean Sacrenf05de732013-02-11 13:30:38 +0000108 * skb_panic - private function for out-of-line support
109 * @skb: buffer
110 * @sz: size
111 * @addr: address
James Hogan99d58512013-02-13 11:20:27 +0000112 * @msg: skb_over_panic or skb_under_panic
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113 *
Jean Sacrenf05de732013-02-11 13:30:38 +0000114 * Out-of-line support for skb_put() and skb_push().
115 * Called via the wrapper skb_over_panic() or skb_under_panic().
116 * Keep out of line to prevent kernel bloat.
117 * __builtin_return_address is not used because it is not always reliable.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700118 */
Jean Sacrenf05de732013-02-11 13:30:38 +0000119static void skb_panic(struct sk_buff *skb, unsigned int sz, void *addr,
James Hogan99d58512013-02-13 11:20:27 +0000120 const char msg[])
Linus Torvalds1da177e2005-04-16 15:20:36 -0700121{
Joe Perchese005d192012-05-16 19:58:40 +0000122 pr_emerg("%s: text:%p len:%d put:%d head:%p data:%p tail:%#lx end:%#lx dev:%s\n",
James Hogan99d58512013-02-13 11:20:27 +0000123 msg, addr, skb->len, sz, skb->head, skb->data,
Joe Perchese005d192012-05-16 19:58:40 +0000124 (unsigned long)skb->tail, (unsigned long)skb->end,
125 skb->dev ? skb->dev->name : "<NULL>");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700126 BUG();
127}
128
Jean Sacrenf05de732013-02-11 13:30:38 +0000129static void skb_over_panic(struct sk_buff *skb, unsigned int sz, void *addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700130{
Jean Sacrenf05de732013-02-11 13:30:38 +0000131 skb_panic(skb, sz, addr, __func__);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700132}
133
Jean Sacrenf05de732013-02-11 13:30:38 +0000134static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr)
135{
136 skb_panic(skb, sz, addr, __func__);
137}
Mel Gormanc93bdd02012-07-31 16:44:19 -0700138
139/*
140 * kmalloc_reserve is a wrapper around kmalloc_node_track_caller that tells
141 * the caller if emergency pfmemalloc reserves are being used. If it is and
142 * the socket is later found to be SOCK_MEMALLOC then PFMEMALLOC reserves
143 * may be used. Otherwise, the packet data may be discarded until enough
144 * memory is free
145 */
146#define kmalloc_reserve(size, gfp, node, pfmemalloc) \
147 __kmalloc_reserve(size, gfp, node, _RET_IP_, pfmemalloc)
stephen hemminger61c5e882012-12-28 18:24:28 +0000148
149static void *__kmalloc_reserve(size_t size, gfp_t flags, int node,
150 unsigned long ip, bool *pfmemalloc)
Mel Gormanc93bdd02012-07-31 16:44:19 -0700151{
152 void *obj;
153 bool ret_pfmemalloc = false;
154
155 /*
156 * Try a regular allocation, when that fails and we're not entitled
157 * to the reserves, fail.
158 */
159 obj = kmalloc_node_track_caller(size,
160 flags | __GFP_NOMEMALLOC | __GFP_NOWARN,
161 node);
162 if (obj || !(gfp_pfmemalloc_allowed(flags)))
163 goto out;
164
165 /* Try again but now we are using pfmemalloc reserves */
166 ret_pfmemalloc = true;
167 obj = kmalloc_node_track_caller(size, flags, node);
168
169out:
170 if (pfmemalloc)
171 *pfmemalloc = ret_pfmemalloc;
172
173 return obj;
174}
175
Linus Torvalds1da177e2005-04-16 15:20:36 -0700176/* Allocate a new skbuff. We do this ourselves so we can fill in a few
177 * 'private' fields and also do memory statistics to find all the
178 * [BEEP] leaks.
179 *
180 */
181
Patrick McHardy0ebd0ac2013-04-17 06:46:58 +0000182struct sk_buff *__alloc_skb_head(gfp_t gfp_mask, int node)
183{
184 struct sk_buff *skb;
185
186 /* Get the HEAD */
187 skb = kmem_cache_alloc_node(skbuff_head_cache,
188 gfp_mask & ~__GFP_DMA, node);
189 if (!skb)
190 goto out;
191
192 /*
193 * Only clear those fields we need to clear, not those that we will
194 * actually initialise below. Hence, don't put any more fields after
195 * the tail pointer in struct sk_buff!
196 */
197 memset(skb, 0, offsetof(struct sk_buff, tail));
Pablo Neira5e71d9d2013-06-03 09:28:43 +0000198 skb->head = NULL;
Patrick McHardy0ebd0ac2013-04-17 06:46:58 +0000199 skb->truesize = sizeof(struct sk_buff);
200 atomic_set(&skb->users, 1);
201
Cong Wang35d04612013-05-29 15:16:05 +0800202 skb->mac_header = (typeof(skb->mac_header))~0U;
Patrick McHardy0ebd0ac2013-04-17 06:46:58 +0000203out:
204 return skb;
205}
206
Linus Torvalds1da177e2005-04-16 15:20:36 -0700207/**
David S. Millerd179cd12005-08-17 14:57:30 -0700208 * __alloc_skb - allocate a network buffer
Linus Torvalds1da177e2005-04-16 15:20:36 -0700209 * @size: size to allocate
210 * @gfp_mask: allocation mask
Mel Gormanc93bdd02012-07-31 16:44:19 -0700211 * @flags: If SKB_ALLOC_FCLONE is set, allocate from fclone cache
212 * instead of head cache and allocate a cloned (child) skb.
213 * If SKB_ALLOC_RX is set, __GFP_MEMALLOC will be used for
214 * allocations in case the data is required for writeback
Christoph Hellwigb30973f2006-12-06 20:32:36 -0800215 * @node: numa node to allocate memory on
Linus Torvalds1da177e2005-04-16 15:20:36 -0700216 *
217 * Allocate a new &sk_buff. The returned buffer has no headroom and a
Ben Hutchings94b60422012-06-06 15:23:37 +0000218 * tail room of at least size bytes. The object has a reference count
219 * of one. The return is the buffer. On a failure the return is %NULL.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700220 *
221 * Buffers may only be allocated from interrupts using a @gfp_mask of
222 * %GFP_ATOMIC.
223 */
Al Virodd0fc662005-10-07 07:46:04 +0100224struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
Mel Gormanc93bdd02012-07-31 16:44:19 -0700225 int flags, int node)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700226{
Christoph Lametere18b8902006-12-06 20:33:20 -0800227 struct kmem_cache *cache;
Benjamin LaHaise4947d3e2006-01-03 14:06:50 -0800228 struct skb_shared_info *shinfo;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700229 struct sk_buff *skb;
230 u8 *data;
Mel Gormanc93bdd02012-07-31 16:44:19 -0700231 bool pfmemalloc;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700232
Mel Gormanc93bdd02012-07-31 16:44:19 -0700233 cache = (flags & SKB_ALLOC_FCLONE)
234 ? skbuff_fclone_cache : skbuff_head_cache;
235
236 if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX))
237 gfp_mask |= __GFP_MEMALLOC;
Herbert Xu8798b3f2006-01-23 16:32:45 -0800238
Linus Torvalds1da177e2005-04-16 15:20:36 -0700239 /* Get the HEAD */
Christoph Hellwigb30973f2006-12-06 20:32:36 -0800240 skb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700241 if (!skb)
242 goto out;
Eric Dumazetec7d2f22010-05-05 01:07:37 -0700243 prefetchw(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700244
Eric Dumazet87fb4b72011-10-13 07:28:54 +0000245 /* We do our best to align skb_shared_info on a separate cache
246 * line. It usually works because kmalloc(X > SMP_CACHE_BYTES) gives
247 * aligned memory blocks, unless SLUB/SLAB debug is enabled.
248 * Both skb->head and skb_shared_info are cache line aligned.
249 */
Tony Lindgrenbc417e32011-11-02 13:40:28 +0000250 size = SKB_DATA_ALIGN(size);
Eric Dumazet87fb4b72011-10-13 07:28:54 +0000251 size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
Mel Gormanc93bdd02012-07-31 16:44:19 -0700252 data = kmalloc_reserve(size, gfp_mask, node, &pfmemalloc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700253 if (!data)
254 goto nodata;
Eric Dumazet87fb4b72011-10-13 07:28:54 +0000255 /* kmalloc(size) might give us more room than requested.
256 * Put skb_shared_info exactly at the end of allocated zone,
257 * to allow max possible filling before reallocation.
258 */
259 size = SKB_WITH_OVERHEAD(ksize(data));
Eric Dumazetec7d2f22010-05-05 01:07:37 -0700260 prefetchw(data + size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700261
Arnaldo Carvalho de Meloca0605a2007-03-19 10:48:59 -0300262 /*
Johannes Bergc8005782008-05-03 20:56:42 -0700263 * Only clear those fields we need to clear, not those that we will
264 * actually initialise below. Hence, don't put any more fields after
265 * the tail pointer in struct sk_buff!
Arnaldo Carvalho de Meloca0605a2007-03-19 10:48:59 -0300266 */
267 memset(skb, 0, offsetof(struct sk_buff, tail));
Eric Dumazet87fb4b72011-10-13 07:28:54 +0000268 /* Account for allocated memory : skb + skb->head */
269 skb->truesize = SKB_TRUESIZE(size);
Mel Gormanc93bdd02012-07-31 16:44:19 -0700270 skb->pfmemalloc = pfmemalloc;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700271 atomic_set(&skb->users, 1);
272 skb->head = data;
273 skb->data = data;
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -0700274 skb_reset_tail_pointer(skb);
Arnaldo Carvalho de Melo4305b542007-04-19 20:43:29 -0700275 skb->end = skb->tail + size;
Cong Wang35d04612013-05-29 15:16:05 +0800276 skb->mac_header = (typeof(skb->mac_header))~0U;
277 skb->transport_header = (typeof(skb->transport_header))~0U;
Stephen Hemminger19633e12009-06-17 05:23:27 +0000278
Benjamin LaHaise4947d3e2006-01-03 14:06:50 -0800279 /* make sure we initialize shinfo sequentially */
280 shinfo = skb_shinfo(skb);
Eric Dumazetec7d2f22010-05-05 01:07:37 -0700281 memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
Benjamin LaHaise4947d3e2006-01-03 14:06:50 -0800282 atomic_set(&shinfo->dataref, 1);
Eric Dumazetc2aa3662011-01-25 23:18:38 +0000283 kmemcheck_annotate_variable(shinfo->destructor_arg);
Benjamin LaHaise4947d3e2006-01-03 14:06:50 -0800284
Mel Gormanc93bdd02012-07-31 16:44:19 -0700285 if (flags & SKB_ALLOC_FCLONE) {
David S. Millerd179cd12005-08-17 14:57:30 -0700286 struct sk_buff *child = skb + 1;
287 atomic_t *fclone_ref = (atomic_t *) (child + 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700288
Vegard Nossumfe55f6d2008-08-30 12:16:35 +0200289 kmemcheck_annotate_bitfield(child, flags1);
290 kmemcheck_annotate_bitfield(child, flags2);
David S. Millerd179cd12005-08-17 14:57:30 -0700291 skb->fclone = SKB_FCLONE_ORIG;
292 atomic_set(fclone_ref, 1);
293
294 child->fclone = SKB_FCLONE_UNAVAILABLE;
Mel Gormanc93bdd02012-07-31 16:44:19 -0700295 child->pfmemalloc = pfmemalloc;
David S. Millerd179cd12005-08-17 14:57:30 -0700296 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700297out:
298 return skb;
299nodata:
Herbert Xu8798b3f2006-01-23 16:32:45 -0800300 kmem_cache_free(cache, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700301 skb = NULL;
302 goto out;
303}
David S. Millerb4ac530fc2009-02-10 02:09:24 -0800304EXPORT_SYMBOL(__alloc_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700305
306/**
Eric Dumazetb2b5ce92011-11-14 06:03:34 +0000307 * build_skb - build a network buffer
308 * @data: data buffer provided by caller
Eric Dumazetd3836f22012-04-27 00:33:38 +0000309 * @frag_size: size of fragment, or 0 if head was kmalloced
Eric Dumazetb2b5ce92011-11-14 06:03:34 +0000310 *
311 * Allocate a new &sk_buff. Caller provides space holding head and
Florian Fainellideceb4c2013-07-23 20:22:39 +0100312 * skb_shared_info. @data must have been allocated by kmalloc() only if
313 * @frag_size is 0, otherwise data should come from the page allocator.
Eric Dumazetb2b5ce92011-11-14 06:03:34 +0000314 * The return is the new skb buffer.
315 * On a failure the return is %NULL, and @data is not freed.
316 * Notes :
317 * Before IO, driver allocates only data buffer where NIC put incoming frame
318 * Driver should add room at head (NET_SKB_PAD) and
319 * MUST add room at tail (SKB_DATA_ALIGN(skb_shared_info))
320 * After IO, driver calls build_skb(), to allocate sk_buff and populate it
321 * before giving packet to stack.
322 * RX rings only contains data buffers, not full skbs.
323 */
Eric Dumazetd3836f22012-04-27 00:33:38 +0000324struct sk_buff *build_skb(void *data, unsigned int frag_size)
Eric Dumazetb2b5ce92011-11-14 06:03:34 +0000325{
326 struct skb_shared_info *shinfo;
327 struct sk_buff *skb;
Eric Dumazetd3836f22012-04-27 00:33:38 +0000328 unsigned int size = frag_size ? : ksize(data);
Eric Dumazetb2b5ce92011-11-14 06:03:34 +0000329
330 skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC);
331 if (!skb)
332 return NULL;
333
Eric Dumazetd3836f22012-04-27 00:33:38 +0000334 size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
Eric Dumazetb2b5ce92011-11-14 06:03:34 +0000335
336 memset(skb, 0, offsetof(struct sk_buff, tail));
337 skb->truesize = SKB_TRUESIZE(size);
Eric Dumazetd3836f22012-04-27 00:33:38 +0000338 skb->head_frag = frag_size != 0;
Eric Dumazetb2b5ce92011-11-14 06:03:34 +0000339 atomic_set(&skb->users, 1);
340 skb->head = data;
341 skb->data = data;
342 skb_reset_tail_pointer(skb);
343 skb->end = skb->tail + size;
Cong Wang35d04612013-05-29 15:16:05 +0800344 skb->mac_header = (typeof(skb->mac_header))~0U;
345 skb->transport_header = (typeof(skb->transport_header))~0U;
Eric Dumazetb2b5ce92011-11-14 06:03:34 +0000346
347 /* make sure we initialize shinfo sequentially */
348 shinfo = skb_shinfo(skb);
349 memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
350 atomic_set(&shinfo->dataref, 1);
351 kmemcheck_annotate_variable(shinfo->destructor_arg);
352
353 return skb;
354}
355EXPORT_SYMBOL(build_skb);
356
Eric Dumazeta1c7fff2012-05-17 07:34:16 +0000357struct netdev_alloc_cache {
Eric Dumazet69b08f62012-09-26 06:46:57 +0000358 struct page_frag frag;
359 /* we maintain a pagecount bias, so that we dont dirty cache line
360 * containing page->_count every time we allocate a fragment.
361 */
362 unsigned int pagecnt_bias;
Eric Dumazeta1c7fff2012-05-17 07:34:16 +0000363};
364static DEFINE_PER_CPU(struct netdev_alloc_cache, netdev_alloc_cache);
365
Mel Gormanc93bdd02012-07-31 16:44:19 -0700366static void *__netdev_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
Eric Dumazet6f532612012-05-18 05:12:12 +0000367{
368 struct netdev_alloc_cache *nc;
369 void *data = NULL;
Eric Dumazet69b08f62012-09-26 06:46:57 +0000370 int order;
Eric Dumazet6f532612012-05-18 05:12:12 +0000371 unsigned long flags;
372
373 local_irq_save(flags);
374 nc = &__get_cpu_var(netdev_alloc_cache);
Eric Dumazet69b08f62012-09-26 06:46:57 +0000375 if (unlikely(!nc->frag.page)) {
Eric Dumazet6f532612012-05-18 05:12:12 +0000376refill:
Eric Dumazet69b08f62012-09-26 06:46:57 +0000377 for (order = NETDEV_FRAG_PAGE_MAX_ORDER; ;) {
378 gfp_t gfp = gfp_mask;
379
380 if (order)
381 gfp |= __GFP_COMP | __GFP_NOWARN;
382 nc->frag.page = alloc_pages(gfp, order);
383 if (likely(nc->frag.page))
384 break;
385 if (--order < 0)
386 goto end;
387 }
388 nc->frag.size = PAGE_SIZE << order;
Alexander Duyck540eb7b2012-07-12 14:23:50 +0000389recycle:
Eric Dumazet69b08f62012-09-26 06:46:57 +0000390 atomic_set(&nc->frag.page->_count, NETDEV_PAGECNT_MAX_BIAS);
391 nc->pagecnt_bias = NETDEV_PAGECNT_MAX_BIAS;
392 nc->frag.offset = 0;
Eric Dumazet6f532612012-05-18 05:12:12 +0000393 }
Alexander Duyck540eb7b2012-07-12 14:23:50 +0000394
Eric Dumazet69b08f62012-09-26 06:46:57 +0000395 if (nc->frag.offset + fragsz > nc->frag.size) {
Alexander Duyck540eb7b2012-07-12 14:23:50 +0000396 /* avoid unnecessary locked operations if possible */
Eric Dumazet69b08f62012-09-26 06:46:57 +0000397 if ((atomic_read(&nc->frag.page->_count) == nc->pagecnt_bias) ||
398 atomic_sub_and_test(nc->pagecnt_bias, &nc->frag.page->_count))
Alexander Duyck540eb7b2012-07-12 14:23:50 +0000399 goto recycle;
400 goto refill;
Eric Dumazet6f532612012-05-18 05:12:12 +0000401 }
Alexander Duyck540eb7b2012-07-12 14:23:50 +0000402
Eric Dumazet69b08f62012-09-26 06:46:57 +0000403 data = page_address(nc->frag.page) + nc->frag.offset;
404 nc->frag.offset += fragsz;
Alexander Duyck540eb7b2012-07-12 14:23:50 +0000405 nc->pagecnt_bias--;
406end:
Eric Dumazet6f532612012-05-18 05:12:12 +0000407 local_irq_restore(flags);
408 return data;
409}
Mel Gormanc93bdd02012-07-31 16:44:19 -0700410
411/**
412 * netdev_alloc_frag - allocate a page fragment
413 * @fragsz: fragment size
414 *
415 * Allocates a frag from a page for receive buffer.
416 * Uses GFP_ATOMIC allocations.
417 */
418void *netdev_alloc_frag(unsigned int fragsz)
419{
420 return __netdev_alloc_frag(fragsz, GFP_ATOMIC | __GFP_COLD);
421}
Eric Dumazet6f532612012-05-18 05:12:12 +0000422EXPORT_SYMBOL(netdev_alloc_frag);
423
424/**
Christoph Hellwig8af27452006-07-31 22:35:23 -0700425 * __netdev_alloc_skb - allocate an skbuff for rx on a specific device
426 * @dev: network device to receive on
427 * @length: length to allocate
428 * @gfp_mask: get_free_pages mask, passed to alloc_skb
429 *
430 * Allocate a new &sk_buff and assign it a usage count of one. The
431 * buffer has unspecified headroom built in. Users should allocate
432 * the headroom they think they need without accounting for the
433 * built in space. The built in space is used for optimisations.
434 *
435 * %NULL is returned if there is no free memory.
436 */
437struct sk_buff *__netdev_alloc_skb(struct net_device *dev,
Eric Dumazet6f532612012-05-18 05:12:12 +0000438 unsigned int length, gfp_t gfp_mask)
Christoph Hellwig8af27452006-07-31 22:35:23 -0700439{
Eric Dumazet6f532612012-05-18 05:12:12 +0000440 struct sk_buff *skb = NULL;
Eric Dumazeta1c7fff2012-05-17 07:34:16 +0000441 unsigned int fragsz = SKB_DATA_ALIGN(length + NET_SKB_PAD) +
442 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
Christoph Hellwig8af27452006-07-31 22:35:23 -0700443
Eric Dumazet310e1582012-07-16 13:15:52 +0200444 if (fragsz <= PAGE_SIZE && !(gfp_mask & (__GFP_WAIT | GFP_DMA))) {
Mel Gormanc93bdd02012-07-31 16:44:19 -0700445 void *data;
446
447 if (sk_memalloc_socks())
448 gfp_mask |= __GFP_MEMALLOC;
449
450 data = __netdev_alloc_frag(fragsz, gfp_mask);
Eric Dumazeta1c7fff2012-05-17 07:34:16 +0000451
Eric Dumazet6f532612012-05-18 05:12:12 +0000452 if (likely(data)) {
453 skb = build_skb(data, fragsz);
454 if (unlikely(!skb))
455 put_page(virt_to_head_page(data));
Eric Dumazeta1c7fff2012-05-17 07:34:16 +0000456 }
Eric Dumazeta1c7fff2012-05-17 07:34:16 +0000457 } else {
Mel Gormanc93bdd02012-07-31 16:44:19 -0700458 skb = __alloc_skb(length + NET_SKB_PAD, gfp_mask,
459 SKB_ALLOC_RX, NUMA_NO_NODE);
Eric Dumazeta1c7fff2012-05-17 07:34:16 +0000460 }
Christoph Hellwig7b2e4972006-08-07 16:09:04 -0700461 if (likely(skb)) {
Christoph Hellwig8af27452006-07-31 22:35:23 -0700462 skb_reserve(skb, NET_SKB_PAD);
Christoph Hellwig7b2e4972006-08-07 16:09:04 -0700463 skb->dev = dev;
464 }
Christoph Hellwig8af27452006-07-31 22:35:23 -0700465 return skb;
466}
David S. Millerb4ac530fc2009-02-10 02:09:24 -0800467EXPORT_SYMBOL(__netdev_alloc_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700468
Peter Zijlstra654bed12008-10-07 14:22:33 -0700469void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off,
Eric Dumazet50269e12012-03-23 23:59:33 +0000470 int size, unsigned int truesize)
Peter Zijlstra654bed12008-10-07 14:22:33 -0700471{
472 skb_fill_page_desc(skb, i, page, off, size);
473 skb->len += size;
474 skb->data_len += size;
Eric Dumazet50269e12012-03-23 23:59:33 +0000475 skb->truesize += truesize;
Peter Zijlstra654bed12008-10-07 14:22:33 -0700476}
477EXPORT_SYMBOL(skb_add_rx_frag);
478
Jason Wangf8e617e2013-11-01 14:07:47 +0800479void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size,
480 unsigned int truesize)
481{
482 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
483
484 skb_frag_size_add(frag, size);
485 skb->len += size;
486 skb->data_len += size;
487 skb->truesize += truesize;
488}
489EXPORT_SYMBOL(skb_coalesce_rx_frag);
490
Herbert Xu27b437c2006-07-13 19:26:39 -0700491static void skb_drop_list(struct sk_buff **listp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700492{
Eric Dumazetbd8a7032013-06-24 06:26:00 -0700493 kfree_skb_list(*listp);
Herbert Xu27b437c2006-07-13 19:26:39 -0700494 *listp = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700495}
496
Herbert Xu27b437c2006-07-13 19:26:39 -0700497static inline void skb_drop_fraglist(struct sk_buff *skb)
498{
499 skb_drop_list(&skb_shinfo(skb)->frag_list);
500}
501
Linus Torvalds1da177e2005-04-16 15:20:36 -0700502static void skb_clone_fraglist(struct sk_buff *skb)
503{
504 struct sk_buff *list;
505
David S. Millerfbb398a2009-06-09 00:18:59 -0700506 skb_walk_frags(skb, list)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700507 skb_get(list);
508}
509
Eric Dumazetd3836f22012-04-27 00:33:38 +0000510static void skb_free_head(struct sk_buff *skb)
511{
512 if (skb->head_frag)
513 put_page(virt_to_head_page(skb->head));
514 else
515 kfree(skb->head);
516}
517
Adrian Bunk5bba1712006-06-29 13:02:35 -0700518static void skb_release_data(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700519{
520 if (!skb->cloned ||
521 !atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1,
522 &skb_shinfo(skb)->dataref)) {
523 if (skb_shinfo(skb)->nr_frags) {
524 int i;
525 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
Ian Campbellea2ab692011-08-22 23:44:58 +0000526 skb_frag_unref(skb, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700527 }
528
Shirley Maa6686f22011-07-06 12:22:12 +0000529 /*
530 * If skb buf is from userspace, we need to notify the caller
531 * the lower device DMA has done;
532 */
533 if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) {
534 struct ubuf_info *uarg;
535
536 uarg = skb_shinfo(skb)->destructor_arg;
537 if (uarg->callback)
Michael S. Tsirkine19d6762012-11-01 09:16:22 +0000538 uarg->callback(uarg, true);
Shirley Maa6686f22011-07-06 12:22:12 +0000539 }
540
David S. Miller21dc3302010-08-23 00:13:46 -0700541 if (skb_has_frag_list(skb))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700542 skb_drop_fraglist(skb);
543
Eric Dumazetd3836f22012-04-27 00:33:38 +0000544 skb_free_head(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700545 }
546}
547
548/*
549 * Free an skbuff by memory without cleaning the state.
550 */
Herbert Xu2d4baff2007-11-26 23:11:19 +0800551static void kfree_skbmem(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700552{
David S. Millerd179cd12005-08-17 14:57:30 -0700553 struct sk_buff *other;
554 atomic_t *fclone_ref;
555
David S. Millerd179cd12005-08-17 14:57:30 -0700556 switch (skb->fclone) {
557 case SKB_FCLONE_UNAVAILABLE:
558 kmem_cache_free(skbuff_head_cache, skb);
559 break;
560
561 case SKB_FCLONE_ORIG:
562 fclone_ref = (atomic_t *) (skb + 2);
563 if (atomic_dec_and_test(fclone_ref))
564 kmem_cache_free(skbuff_fclone_cache, skb);
565 break;
566
567 case SKB_FCLONE_CLONE:
568 fclone_ref = (atomic_t *) (skb + 1);
569 other = skb - 1;
570
571 /* The clone portion is available for
572 * fast-cloning again.
573 */
574 skb->fclone = SKB_FCLONE_UNAVAILABLE;
575
576 if (atomic_dec_and_test(fclone_ref))
577 kmem_cache_free(skbuff_fclone_cache, other);
578 break;
Stephen Hemminger3ff50b72007-04-20 17:09:22 -0700579 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700580}
581
Lennert Buytenhek04a4bb52008-10-01 02:33:12 -0700582static void skb_release_head_state(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700583{
Eric Dumazetadf30902009-06-02 05:19:30 +0000584 skb_dst_drop(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700585#ifdef CONFIG_XFRM
586 secpath_put(skb->sp);
587#endif
Stephen Hemminger9c2b3322005-04-19 22:39:42 -0700588 if (skb->destructor) {
589 WARN_ON(in_irq());
Linus Torvalds1da177e2005-04-16 15:20:36 -0700590 skb->destructor(skb);
591 }
Igor Maravića3bf7ae2011-12-12 02:58:22 +0000592#if IS_ENABLED(CONFIG_NF_CONNTRACK)
Yasuyuki Kozakai5f79e0f2007-03-23 11:17:07 -0700593 nf_conntrack_put(skb->nfct);
KOVACS Krisztian2fc72c72011-01-12 20:25:08 +0100594#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700595#ifdef CONFIG_BRIDGE_NETFILTER
596 nf_bridge_put(skb->nf_bridge);
597#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700598/* XXX: IS this still necessary? - JHS */
599#ifdef CONFIG_NET_SCHED
600 skb->tc_index = 0;
601#ifdef CONFIG_NET_CLS_ACT
602 skb->tc_verd = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700603#endif
604#endif
Lennert Buytenhek04a4bb52008-10-01 02:33:12 -0700605}
606
607/* Free everything but the sk_buff shell. */
608static void skb_release_all(struct sk_buff *skb)
609{
610 skb_release_head_state(skb);
Pablo Neira5e71d9d2013-06-03 09:28:43 +0000611 if (likely(skb->head))
Patrick McHardy0ebd0ac2013-04-17 06:46:58 +0000612 skb_release_data(skb);
Herbert Xu2d4baff2007-11-26 23:11:19 +0800613}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700614
Herbert Xu2d4baff2007-11-26 23:11:19 +0800615/**
616 * __kfree_skb - private function
617 * @skb: buffer
618 *
619 * Free an sk_buff. Release anything attached to the buffer.
620 * Clean the state. This is an internal helper function. Users should
621 * always call kfree_skb
622 */
623
624void __kfree_skb(struct sk_buff *skb)
625{
626 skb_release_all(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700627 kfree_skbmem(skb);
628}
David S. Millerb4ac530fc2009-02-10 02:09:24 -0800629EXPORT_SYMBOL(__kfree_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700630
631/**
Jörn Engel231d06a2006-03-20 21:28:35 -0800632 * kfree_skb - free an sk_buff
633 * @skb: buffer to free
634 *
635 * Drop a reference to the buffer and free it if the usage count has
636 * hit zero.
637 */
638void kfree_skb(struct sk_buff *skb)
639{
640 if (unlikely(!skb))
641 return;
642 if (likely(atomic_read(&skb->users) == 1))
643 smp_rmb();
644 else if (likely(!atomic_dec_and_test(&skb->users)))
645 return;
Neil Hormanead2ceb2009-03-11 09:49:55 +0000646 trace_kfree_skb(skb, __builtin_return_address(0));
Jörn Engel231d06a2006-03-20 21:28:35 -0800647 __kfree_skb(skb);
648}
David S. Millerb4ac530fc2009-02-10 02:09:24 -0800649EXPORT_SYMBOL(kfree_skb);
Jörn Engel231d06a2006-03-20 21:28:35 -0800650
Eric Dumazetbd8a7032013-06-24 06:26:00 -0700651void kfree_skb_list(struct sk_buff *segs)
652{
653 while (segs) {
654 struct sk_buff *next = segs->next;
655
656 kfree_skb(segs);
657 segs = next;
658 }
659}
660EXPORT_SYMBOL(kfree_skb_list);
661
Stephen Hemmingerd1a203e2008-11-01 21:01:09 -0700662/**
Michael S. Tsirkin25121172012-11-01 09:16:28 +0000663 * skb_tx_error - report an sk_buff xmit error
664 * @skb: buffer that triggered an error
665 *
666 * Report xmit error if a device callback is tracking this skb.
667 * skb must be freed afterwards.
668 */
669void skb_tx_error(struct sk_buff *skb)
670{
671 if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) {
672 struct ubuf_info *uarg;
673
674 uarg = skb_shinfo(skb)->destructor_arg;
675 if (uarg->callback)
676 uarg->callback(uarg, false);
677 skb_shinfo(skb)->tx_flags &= ~SKBTX_DEV_ZEROCOPY;
678 }
679}
680EXPORT_SYMBOL(skb_tx_error);
681
682/**
Neil Hormanead2ceb2009-03-11 09:49:55 +0000683 * consume_skb - free an skbuff
684 * @skb: buffer to free
685 *
686 * Drop a ref to the buffer and free it if the usage count has hit zero
687 * Functions identically to kfree_skb, but kfree_skb assumes that the frame
688 * is being dropped after a failure and notes that
689 */
690void consume_skb(struct sk_buff *skb)
691{
692 if (unlikely(!skb))
693 return;
694 if (likely(atomic_read(&skb->users) == 1))
695 smp_rmb();
696 else if (likely(!atomic_dec_and_test(&skb->users)))
697 return;
Koki Sanagi07dc22e2010-08-23 18:46:12 +0900698 trace_consume_skb(skb);
Neil Hormanead2ceb2009-03-11 09:49:55 +0000699 __kfree_skb(skb);
700}
701EXPORT_SYMBOL(consume_skb);
702
Herbert Xudec18812007-10-14 00:37:30 -0700703static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
704{
705 new->tstamp = old->tstamp;
706 new->dev = old->dev;
707 new->transport_header = old->transport_header;
708 new->network_header = old->network_header;
709 new->mac_header = old->mac_header;
Joe Stringer4bc41b82013-07-03 16:04:25 +0900710 new->inner_protocol = old->inner_protocol;
Joseph Gasparakis6a674e92012-12-07 14:14:14 +0000711 new->inner_transport_header = old->inner_transport_header;
Pravin B Shelar92df9b22013-02-01 15:18:49 +0000712 new->inner_network_header = old->inner_network_header;
Pravin B Shelaraefbd2b2013-03-07 13:21:46 +0000713 new->inner_mac_header = old->inner_mac_header;
Eric Dumazet7fee2262010-05-11 23:19:48 +0000714 skb_dst_copy(new, old);
Tom Herbert0a9627f2010-03-16 08:03:29 +0000715 new->rxhash = old->rxhash;
Changli Gao6461be32011-08-19 04:44:18 +0000716 new->ooo_okay = old->ooo_okay;
Tom Herbertbdeab992011-08-14 19:45:55 +0000717 new->l4_rxhash = old->l4_rxhash;
Ben Greear3bdc0eb2012-02-11 15:39:30 +0000718 new->no_fcs = old->no_fcs;
Joseph Gasparakis6a674e92012-12-07 14:14:14 +0000719 new->encapsulation = old->encapsulation;
Alexey Dobriyandef8b4f2008-10-28 13:24:06 -0700720#ifdef CONFIG_XFRM
Herbert Xudec18812007-10-14 00:37:30 -0700721 new->sp = secpath_get(old->sp);
722#endif
723 memcpy(new->cb, old->cb, sizeof(old->cb));
Herbert Xu9bcb97c2009-05-22 22:20:02 +0000724 new->csum = old->csum;
Herbert Xudec18812007-10-14 00:37:30 -0700725 new->local_df = old->local_df;
726 new->pkt_type = old->pkt_type;
727 new->ip_summed = old->ip_summed;
728 skb_copy_queue_mapping(new, old);
729 new->priority = old->priority;
Igor Maravića3bf7ae2011-12-12 02:58:22 +0000730#if IS_ENABLED(CONFIG_IP_VS)
Herbert Xudec18812007-10-14 00:37:30 -0700731 new->ipvs_property = old->ipvs_property;
732#endif
Mel Gormanc93bdd02012-07-31 16:44:19 -0700733 new->pfmemalloc = old->pfmemalloc;
Herbert Xudec18812007-10-14 00:37:30 -0700734 new->protocol = old->protocol;
735 new->mark = old->mark;
Eric Dumazet8964be42009-11-20 15:35:04 -0800736 new->skb_iif = old->skb_iif;
Herbert Xudec18812007-10-14 00:37:30 -0700737 __nf_copy(new, old);
Igor Maravića3bf7ae2011-12-12 02:58:22 +0000738#if IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TRACE)
Herbert Xudec18812007-10-14 00:37:30 -0700739 new->nf_trace = old->nf_trace;
740#endif
741#ifdef CONFIG_NET_SCHED
742 new->tc_index = old->tc_index;
743#ifdef CONFIG_NET_CLS_ACT
744 new->tc_verd = old->tc_verd;
745#endif
746#endif
Patrick McHardy86a9bad2013-04-19 02:04:30 +0000747 new->vlan_proto = old->vlan_proto;
Patrick McHardy6aa895b02008-07-14 22:49:06 -0700748 new->vlan_tci = old->vlan_tci;
749
Herbert Xudec18812007-10-14 00:37:30 -0700750 skb_copy_secmark(new, old);
Eliezer Tamir06021292013-06-10 11:39:50 +0300751
Cong Wange0d10952013-08-01 11:10:25 +0800752#ifdef CONFIG_NET_RX_BUSY_POLL
Eliezer Tamir06021292013-06-10 11:39:50 +0300753 new->napi_id = old->napi_id;
754#endif
Herbert Xudec18812007-10-14 00:37:30 -0700755}
756
Herbert Xu82c49a32009-05-22 22:11:37 +0000757/*
758 * You should not add any new code to this function. Add it to
759 * __copy_skb_header above instead.
760 */
Herbert Xue0053ec2007-10-14 00:37:52 -0700761static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700762{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700763#define C(x) n->x = skb->x
764
765 n->next = n->prev = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700766 n->sk = NULL;
Herbert Xudec18812007-10-14 00:37:30 -0700767 __copy_skb_header(n, skb);
768
Linus Torvalds1da177e2005-04-16 15:20:36 -0700769 C(len);
770 C(data_len);
Alexey Dobriyan3e6b3b22007-03-16 15:00:46 -0700771 C(mac_len);
Patrick McHardy334a8132007-06-25 04:35:20 -0700772 n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len;
Paul Moore02f1c892008-01-07 21:56:41 -0800773 n->cloned = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700774 n->nohdr = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700775 n->destructor = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700776 C(tail);
777 C(end);
Paul Moore02f1c892008-01-07 21:56:41 -0800778 C(head);
Eric Dumazetd3836f22012-04-27 00:33:38 +0000779 C(head_frag);
Paul Moore02f1c892008-01-07 21:56:41 -0800780 C(data);
781 C(truesize);
782 atomic_set(&n->users, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700783
784 atomic_inc(&(skb_shinfo(skb)->dataref));
785 skb->cloned = 1;
786
787 return n;
Herbert Xue0053ec2007-10-14 00:37:52 -0700788#undef C
789}
790
791/**
792 * skb_morph - morph one skb into another
793 * @dst: the skb to receive the contents
794 * @src: the skb to supply the contents
795 *
796 * This is identical to skb_clone except that the target skb is
797 * supplied by the user.
798 *
799 * The target skb is returned upon exit.
800 */
801struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src)
802{
Herbert Xu2d4baff2007-11-26 23:11:19 +0800803 skb_release_all(dst);
Herbert Xue0053ec2007-10-14 00:37:52 -0700804 return __skb_clone(dst, src);
805}
806EXPORT_SYMBOL_GPL(skb_morph);
807
Ben Hutchings2c530402012-07-10 10:55:09 +0000808/**
809 * skb_copy_ubufs - copy userspace skb frags buffers to kernel
Michael S. Tsirkin48c83012011-08-31 08:03:29 +0000810 * @skb: the skb to modify
811 * @gfp_mask: allocation priority
812 *
813 * This must be called on SKBTX_DEV_ZEROCOPY skb.
814 * It will copy all frags into kernel and drop the reference
815 * to userspace pages.
816 *
817 * If this function is called from an interrupt gfp_mask() must be
818 * %GFP_ATOMIC.
819 *
820 * Returns 0 on success or a negative error code on failure
821 * to allocate kernel memory to copy to.
822 */
823int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask)
Shirley Maa6686f22011-07-06 12:22:12 +0000824{
825 int i;
826 int num_frags = skb_shinfo(skb)->nr_frags;
827 struct page *page, *head = NULL;
828 struct ubuf_info *uarg = skb_shinfo(skb)->destructor_arg;
829
830 for (i = 0; i < num_frags; i++) {
831 u8 *vaddr;
832 skb_frag_t *f = &skb_shinfo(skb)->frags[i];
833
Krishna Kumar02756ed2012-07-17 02:05:29 +0000834 page = alloc_page(gfp_mask);
Shirley Maa6686f22011-07-06 12:22:12 +0000835 if (!page) {
836 while (head) {
Sunghan Suh40dadff2013-07-12 16:17:23 +0900837 struct page *next = (struct page *)page_private(head);
Shirley Maa6686f22011-07-06 12:22:12 +0000838 put_page(head);
839 head = next;
840 }
841 return -ENOMEM;
842 }
Eric Dumazet51c56b02012-04-05 11:35:15 +0200843 vaddr = kmap_atomic(skb_frag_page(f));
Shirley Maa6686f22011-07-06 12:22:12 +0000844 memcpy(page_address(page),
Eric Dumazet9e903e02011-10-18 21:00:24 +0000845 vaddr + f->page_offset, skb_frag_size(f));
Eric Dumazet51c56b02012-04-05 11:35:15 +0200846 kunmap_atomic(vaddr);
Sunghan Suh40dadff2013-07-12 16:17:23 +0900847 set_page_private(page, (unsigned long)head);
Shirley Maa6686f22011-07-06 12:22:12 +0000848 head = page;
849 }
850
851 /* skb frags release userspace buffers */
Krishna Kumar02756ed2012-07-17 02:05:29 +0000852 for (i = 0; i < num_frags; i++)
Ian Campbella8605c62011-10-19 23:01:49 +0000853 skb_frag_unref(skb, i);
Shirley Maa6686f22011-07-06 12:22:12 +0000854
Michael S. Tsirkine19d6762012-11-01 09:16:22 +0000855 uarg->callback(uarg, false);
Shirley Maa6686f22011-07-06 12:22:12 +0000856
857 /* skb frags point to kernel buffers */
Krishna Kumar02756ed2012-07-17 02:05:29 +0000858 for (i = num_frags - 1; i >= 0; i--) {
859 __skb_fill_page_desc(skb, i, head, 0,
860 skb_shinfo(skb)->frags[i].size);
Sunghan Suh40dadff2013-07-12 16:17:23 +0900861 head = (struct page *)page_private(head);
Shirley Maa6686f22011-07-06 12:22:12 +0000862 }
Michael S. Tsirkin48c83012011-08-31 08:03:29 +0000863
864 skb_shinfo(skb)->tx_flags &= ~SKBTX_DEV_ZEROCOPY;
Shirley Maa6686f22011-07-06 12:22:12 +0000865 return 0;
866}
Michael S. Tsirkindcc0fb72012-07-20 09:23:20 +0000867EXPORT_SYMBOL_GPL(skb_copy_ubufs);
Shirley Maa6686f22011-07-06 12:22:12 +0000868
Herbert Xue0053ec2007-10-14 00:37:52 -0700869/**
870 * skb_clone - duplicate an sk_buff
871 * @skb: buffer to clone
872 * @gfp_mask: allocation priority
873 *
874 * Duplicate an &sk_buff. The new one is not owned by a socket. Both
875 * copies share the same packet data but not structure. The new
876 * buffer has a reference count of 1. If the allocation fails the
877 * function returns %NULL otherwise the new buffer is returned.
878 *
879 * If this function is called from an interrupt gfp_mask() must be
880 * %GFP_ATOMIC.
881 */
882
883struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask)
884{
885 struct sk_buff *n;
886
Michael S. Tsirkin70008aa2012-07-20 09:23:10 +0000887 if (skb_orphan_frags(skb, gfp_mask))
888 return NULL;
Shirley Maa6686f22011-07-06 12:22:12 +0000889
Herbert Xue0053ec2007-10-14 00:37:52 -0700890 n = skb + 1;
891 if (skb->fclone == SKB_FCLONE_ORIG &&
892 n->fclone == SKB_FCLONE_UNAVAILABLE) {
893 atomic_t *fclone_ref = (atomic_t *) (n + 1);
894 n->fclone = SKB_FCLONE_CLONE;
895 atomic_inc(fclone_ref);
896 } else {
Mel Gormanc93bdd02012-07-31 16:44:19 -0700897 if (skb_pfmemalloc(skb))
898 gfp_mask |= __GFP_MEMALLOC;
899
Herbert Xue0053ec2007-10-14 00:37:52 -0700900 n = kmem_cache_alloc(skbuff_head_cache, gfp_mask);
901 if (!n)
902 return NULL;
Vegard Nossumfe55f6d2008-08-30 12:16:35 +0200903
904 kmemcheck_annotate_bitfield(n, flags1);
905 kmemcheck_annotate_bitfield(n, flags2);
Herbert Xue0053ec2007-10-14 00:37:52 -0700906 n->fclone = SKB_FCLONE_UNAVAILABLE;
907 }
908
909 return __skb_clone(n, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700910}
David S. Millerb4ac530fc2009-02-10 02:09:24 -0800911EXPORT_SYMBOL(skb_clone);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700912
Pravin B Shelarf5b17292013-03-07 13:21:40 +0000913static void skb_headers_offset_update(struct sk_buff *skb, int off)
914{
Eric Dumazet030737b2013-10-19 11:42:54 -0700915 /* Only adjust this if it actually is csum_start rather than csum */
916 if (skb->ip_summed == CHECKSUM_PARTIAL)
917 skb->csum_start += off;
Pravin B Shelarf5b17292013-03-07 13:21:40 +0000918 /* {transport,network,mac}_header and tail are relative to skb->head */
919 skb->transport_header += off;
920 skb->network_header += off;
921 if (skb_mac_header_was_set(skb))
922 skb->mac_header += off;
923 skb->inner_transport_header += off;
924 skb->inner_network_header += off;
Pravin B Shelaraefbd2b2013-03-07 13:21:46 +0000925 skb->inner_mac_header += off;
Pravin B Shelarf5b17292013-03-07 13:21:40 +0000926}
927
Linus Torvalds1da177e2005-04-16 15:20:36 -0700928static void copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
929{
Herbert Xudec18812007-10-14 00:37:30 -0700930 __copy_skb_header(new, old);
931
Herbert Xu79671682006-06-22 02:40:14 -0700932 skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size;
933 skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs;
934 skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700935}
936
Mel Gormanc93bdd02012-07-31 16:44:19 -0700937static inline int skb_alloc_rx_flag(const struct sk_buff *skb)
938{
939 if (skb_pfmemalloc(skb))
940 return SKB_ALLOC_RX;
941 return 0;
942}
943
Linus Torvalds1da177e2005-04-16 15:20:36 -0700944/**
945 * skb_copy - create private copy of an sk_buff
946 * @skb: buffer to copy
947 * @gfp_mask: allocation priority
948 *
949 * Make a copy of both an &sk_buff and its data. This is used when the
950 * caller wishes to modify the data and needs a private copy of the
951 * data to alter. Returns %NULL on failure or the pointer to the buffer
952 * on success. The returned buffer has a reference count of 1.
953 *
954 * As by-product this function converts non-linear &sk_buff to linear
955 * one, so that &sk_buff becomes completely private and caller is allowed
956 * to modify all the data of returned buffer. This means that this
957 * function is not recommended for use in circumstances when only
958 * header is going to be modified. Use pskb_copy() instead.
959 */
960
Al Virodd0fc662005-10-07 07:46:04 +0100961struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700962{
Eric Dumazet6602ceb2010-09-01 05:25:10 +0000963 int headerlen = skb_headroom(skb);
Alexander Duyckec47ea82012-05-04 14:26:56 +0000964 unsigned int size = skb_end_offset(skb) + skb->data_len;
Mel Gormanc93bdd02012-07-31 16:44:19 -0700965 struct sk_buff *n = __alloc_skb(size, gfp_mask,
966 skb_alloc_rx_flag(skb), NUMA_NO_NODE);
Eric Dumazet6602ceb2010-09-01 05:25:10 +0000967
Linus Torvalds1da177e2005-04-16 15:20:36 -0700968 if (!n)
969 return NULL;
970
971 /* Set the data pointer */
972 skb_reserve(n, headerlen);
973 /* Set the tail pointer and length */
974 skb_put(n, skb->len);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700975
976 if (skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len))
977 BUG();
978
979 copy_skb_header(n, skb);
980 return n;
981}
David S. Millerb4ac530fc2009-02-10 02:09:24 -0800982EXPORT_SYMBOL(skb_copy);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700983
984/**
Eric Dumazet117632e2011-12-03 21:39:53 +0000985 * __pskb_copy - create copy of an sk_buff with private head.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700986 * @skb: buffer to copy
Eric Dumazet117632e2011-12-03 21:39:53 +0000987 * @headroom: headroom of new skb
Linus Torvalds1da177e2005-04-16 15:20:36 -0700988 * @gfp_mask: allocation priority
989 *
990 * Make a copy of both an &sk_buff and part of its data, located
991 * in header. Fragmented data remain shared. This is used when
992 * the caller wishes to modify only header of &sk_buff and needs
993 * private copy of the header to alter. Returns %NULL on failure
994 * or the pointer to the buffer on success.
995 * The returned buffer has a reference count of 1.
996 */
997
Eric Dumazet117632e2011-12-03 21:39:53 +0000998struct sk_buff *__pskb_copy(struct sk_buff *skb, int headroom, gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700999{
Eric Dumazet117632e2011-12-03 21:39:53 +00001000 unsigned int size = skb_headlen(skb) + headroom;
Mel Gormanc93bdd02012-07-31 16:44:19 -07001001 struct sk_buff *n = __alloc_skb(size, gfp_mask,
1002 skb_alloc_rx_flag(skb), NUMA_NO_NODE);
Eric Dumazet6602ceb2010-09-01 05:25:10 +00001003
Linus Torvalds1da177e2005-04-16 15:20:36 -07001004 if (!n)
1005 goto out;
1006
1007 /* Set the data pointer */
Eric Dumazet117632e2011-12-03 21:39:53 +00001008 skb_reserve(n, headroom);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001009 /* Set the tail pointer and length */
1010 skb_put(n, skb_headlen(skb));
1011 /* Copy the bytes */
Arnaldo Carvalho de Melod626f622007-03-27 18:55:52 -03001012 skb_copy_from_linear_data(skb, n->data, n->len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001013
Herbert Xu25f484a2006-11-07 14:57:15 -08001014 n->truesize += skb->data_len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001015 n->data_len = skb->data_len;
1016 n->len = skb->len;
1017
1018 if (skb_shinfo(skb)->nr_frags) {
1019 int i;
1020
Michael S. Tsirkin70008aa2012-07-20 09:23:10 +00001021 if (skb_orphan_frags(skb, gfp_mask)) {
1022 kfree_skb(n);
1023 n = NULL;
1024 goto out;
Shirley Maa6686f22011-07-06 12:22:12 +00001025 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001026 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1027 skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i];
Ian Campbellea2ab692011-08-22 23:44:58 +00001028 skb_frag_ref(skb, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001029 }
1030 skb_shinfo(n)->nr_frags = i;
1031 }
1032
David S. Miller21dc3302010-08-23 00:13:46 -07001033 if (skb_has_frag_list(skb)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001034 skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list;
1035 skb_clone_fraglist(n);
1036 }
1037
1038 copy_skb_header(n, skb);
1039out:
1040 return n;
1041}
Eric Dumazet117632e2011-12-03 21:39:53 +00001042EXPORT_SYMBOL(__pskb_copy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001043
1044/**
1045 * pskb_expand_head - reallocate header of &sk_buff
1046 * @skb: buffer to reallocate
1047 * @nhead: room to add at head
1048 * @ntail: room to add at tail
1049 * @gfp_mask: allocation priority
1050 *
Mathias Krausebc323832013-11-07 14:18:26 +01001051 * Expands (or creates identical copy, if @nhead and @ntail are zero)
1052 * header of @skb. &sk_buff itself is not changed. &sk_buff MUST have
Linus Torvalds1da177e2005-04-16 15:20:36 -07001053 * reference count of 1. Returns zero in the case of success or error,
1054 * if expansion failed. In the last case, &sk_buff is not changed.
1055 *
1056 * All the pointers pointing into skb header may change and must be
1057 * reloaded after call to this function.
1058 */
1059
Victor Fusco86a76ca2005-07-08 14:57:47 -07001060int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
Al Virodd0fc662005-10-07 07:46:04 +01001061 gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001062{
1063 int i;
1064 u8 *data;
Alexander Duyckec47ea82012-05-04 14:26:56 +00001065 int size = nhead + skb_end_offset(skb) + ntail;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001066 long off;
1067
Herbert Xu4edd87a2008-10-01 07:09:38 -07001068 BUG_ON(nhead < 0);
1069
Linus Torvalds1da177e2005-04-16 15:20:36 -07001070 if (skb_shared(skb))
1071 BUG();
1072
1073 size = SKB_DATA_ALIGN(size);
1074
Mel Gormanc93bdd02012-07-31 16:44:19 -07001075 if (skb_pfmemalloc(skb))
1076 gfp_mask |= __GFP_MEMALLOC;
1077 data = kmalloc_reserve(size + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
1078 gfp_mask, NUMA_NO_NODE, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001079 if (!data)
1080 goto nodata;
Eric Dumazet87151b82012-04-10 20:08:39 +00001081 size = SKB_WITH_OVERHEAD(ksize(data));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001082
1083 /* Copy only real data... and, alas, header. This should be
Eric Dumazet6602ceb2010-09-01 05:25:10 +00001084 * optimized for the cases when header is void.
1085 */
1086 memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head);
1087
1088 memcpy((struct skb_shared_info *)(data + size),
1089 skb_shinfo(skb),
Eric Dumazetfed66382010-07-22 19:09:08 +00001090 offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags]));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001091
Alexander Duyck3e245912012-05-04 14:26:51 +00001092 /*
1093 * if shinfo is shared we must drop the old head gracefully, but if it
1094 * is not we can just drop the old head and let the existing refcount
1095 * be since all we did is relocate the values
1096 */
1097 if (skb_cloned(skb)) {
Shirley Maa6686f22011-07-06 12:22:12 +00001098 /* copy this zero copy skb frags */
Michael S. Tsirkin70008aa2012-07-20 09:23:10 +00001099 if (skb_orphan_frags(skb, gfp_mask))
1100 goto nofrags;
Eric Dumazet1fd63042010-09-02 23:09:32 +00001101 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
Ian Campbellea2ab692011-08-22 23:44:58 +00001102 skb_frag_ref(skb, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001103
Eric Dumazet1fd63042010-09-02 23:09:32 +00001104 if (skb_has_frag_list(skb))
1105 skb_clone_fraglist(skb);
1106
1107 skb_release_data(skb);
Alexander Duyck3e245912012-05-04 14:26:51 +00001108 } else {
1109 skb_free_head(skb);
Eric Dumazet1fd63042010-09-02 23:09:32 +00001110 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001111 off = (data + nhead) - skb->head;
1112
1113 skb->head = data;
Eric Dumazetd3836f22012-04-27 00:33:38 +00001114 skb->head_frag = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001115 skb->data += off;
Arnaldo Carvalho de Melo4305b542007-04-19 20:43:29 -07001116#ifdef NET_SKBUFF_DATA_USES_OFFSET
1117 skb->end = size;
Patrick McHardy56eb8882007-04-09 11:45:04 -07001118 off = nhead;
Arnaldo Carvalho de Melo4305b542007-04-19 20:43:29 -07001119#else
1120 skb->end = skb->head + size;
Patrick McHardy56eb8882007-04-09 11:45:04 -07001121#endif
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -07001122 skb->tail += off;
Peter Pan(潘卫平)b41abb42013-06-06 21:27:21 +08001123 skb_headers_offset_update(skb, nhead);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001124 skb->cloned = 0;
Patrick McHardy334a8132007-06-25 04:35:20 -07001125 skb->hdr_len = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001126 skb->nohdr = 0;
1127 atomic_set(&skb_shinfo(skb)->dataref, 1);
1128 return 0;
1129
Shirley Maa6686f22011-07-06 12:22:12 +00001130nofrags:
1131 kfree(data);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001132nodata:
1133 return -ENOMEM;
1134}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08001135EXPORT_SYMBOL(pskb_expand_head);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001136
1137/* Make private copy of skb with writable head and some headroom */
1138
1139struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom)
1140{
1141 struct sk_buff *skb2;
1142 int delta = headroom - skb_headroom(skb);
1143
1144 if (delta <= 0)
1145 skb2 = pskb_copy(skb, GFP_ATOMIC);
1146 else {
1147 skb2 = skb_clone(skb, GFP_ATOMIC);
1148 if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0,
1149 GFP_ATOMIC)) {
1150 kfree_skb(skb2);
1151 skb2 = NULL;
1152 }
1153 }
1154 return skb2;
1155}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08001156EXPORT_SYMBOL(skb_realloc_headroom);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001157
1158/**
1159 * skb_copy_expand - copy and expand sk_buff
1160 * @skb: buffer to copy
1161 * @newheadroom: new free bytes at head
1162 * @newtailroom: new free bytes at tail
1163 * @gfp_mask: allocation priority
1164 *
1165 * Make a copy of both an &sk_buff and its data and while doing so
1166 * allocate additional space.
1167 *
1168 * This is used when the caller wishes to modify the data and needs a
1169 * private copy of the data to alter as well as more space for new fields.
1170 * Returns %NULL on failure or the pointer to the buffer
1171 * on success. The returned buffer has a reference count of 1.
1172 *
1173 * You must pass %GFP_ATOMIC as the allocation priority if this function
1174 * is called from an interrupt.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001175 */
1176struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
Victor Fusco86a76ca2005-07-08 14:57:47 -07001177 int newheadroom, int newtailroom,
Al Virodd0fc662005-10-07 07:46:04 +01001178 gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001179{
1180 /*
1181 * Allocate the copy buffer
1182 */
Mel Gormanc93bdd02012-07-31 16:44:19 -07001183 struct sk_buff *n = __alloc_skb(newheadroom + skb->len + newtailroom,
1184 gfp_mask, skb_alloc_rx_flag(skb),
1185 NUMA_NO_NODE);
Patrick McHardyefd1e8d2007-04-10 18:30:09 -07001186 int oldheadroom = skb_headroom(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001187 int head_copy_len, head_copy_off;
1188
1189 if (!n)
1190 return NULL;
1191
1192 skb_reserve(n, newheadroom);
1193
1194 /* Set the tail pointer and length */
1195 skb_put(n, skb->len);
1196
Patrick McHardyefd1e8d2007-04-10 18:30:09 -07001197 head_copy_len = oldheadroom;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001198 head_copy_off = 0;
1199 if (newheadroom <= head_copy_len)
1200 head_copy_len = newheadroom;
1201 else
1202 head_copy_off = newheadroom - head_copy_len;
1203
1204 /* Copy the linear header and data. */
1205 if (skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off,
1206 skb->len + head_copy_len))
1207 BUG();
1208
1209 copy_skb_header(n, skb);
1210
Eric Dumazet030737b2013-10-19 11:42:54 -07001211 skb_headers_offset_update(n, newheadroom - oldheadroom);
Patrick McHardyefd1e8d2007-04-10 18:30:09 -07001212
Linus Torvalds1da177e2005-04-16 15:20:36 -07001213 return n;
1214}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08001215EXPORT_SYMBOL(skb_copy_expand);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001216
1217/**
1218 * skb_pad - zero pad the tail of an skb
1219 * @skb: buffer to pad
1220 * @pad: space to pad
1221 *
1222 * Ensure that a buffer is followed by a padding area that is zero
1223 * filled. Used by network drivers which may DMA or transfer data
1224 * beyond the buffer end onto the wire.
1225 *
Herbert Xu5b057c62006-06-23 02:06:41 -07001226 * May return error in out of memory cases. The skb is freed on error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001227 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001228
Herbert Xu5b057c62006-06-23 02:06:41 -07001229int skb_pad(struct sk_buff *skb, int pad)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001230{
Herbert Xu5b057c62006-06-23 02:06:41 -07001231 int err;
1232 int ntail;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001233
Linus Torvalds1da177e2005-04-16 15:20:36 -07001234 /* If the skbuff is non linear tailroom is always zero.. */
Herbert Xu5b057c62006-06-23 02:06:41 -07001235 if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001236 memset(skb->data+skb->len, 0, pad);
Herbert Xu5b057c62006-06-23 02:06:41 -07001237 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001238 }
Herbert Xu5b057c62006-06-23 02:06:41 -07001239
Arnaldo Carvalho de Melo4305b542007-04-19 20:43:29 -07001240 ntail = skb->data_len + pad - (skb->end - skb->tail);
Herbert Xu5b057c62006-06-23 02:06:41 -07001241 if (likely(skb_cloned(skb) || ntail > 0)) {
1242 err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC);
1243 if (unlikely(err))
1244 goto free_skb;
1245 }
1246
1247 /* FIXME: The use of this function with non-linear skb's really needs
1248 * to be audited.
1249 */
1250 err = skb_linearize(skb);
1251 if (unlikely(err))
1252 goto free_skb;
1253
1254 memset(skb->data + skb->len, 0, pad);
1255 return 0;
1256
1257free_skb:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001258 kfree_skb(skb);
Herbert Xu5b057c62006-06-23 02:06:41 -07001259 return err;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001260}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08001261EXPORT_SYMBOL(skb_pad);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001262
Ilpo Järvinen0dde3e12008-03-27 17:43:41 -07001263/**
Mathias Krause0c7ddf32013-11-07 14:18:24 +01001264 * pskb_put - add data to the tail of a potentially fragmented buffer
1265 * @skb: start of the buffer to use
1266 * @tail: tail fragment of the buffer to use
1267 * @len: amount of data to add
1268 *
1269 * This function extends the used data area of the potentially
1270 * fragmented buffer. @tail must be the last fragment of @skb -- or
1271 * @skb itself. If this would exceed the total buffer size the kernel
1272 * will panic. A pointer to the first byte of the extra data is
1273 * returned.
1274 */
1275
1276unsigned char *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len)
1277{
1278 if (tail != skb) {
1279 skb->data_len += len;
1280 skb->len += len;
1281 }
1282 return skb_put(tail, len);
1283}
1284EXPORT_SYMBOL_GPL(pskb_put);
1285
1286/**
Ilpo Järvinen0dde3e12008-03-27 17:43:41 -07001287 * skb_put - add data to a buffer
1288 * @skb: buffer to use
1289 * @len: amount of data to add
1290 *
1291 * This function extends the used data area of the buffer. If this would
1292 * exceed the total buffer size the kernel will panic. A pointer to the
1293 * first byte of the extra data is returned.
1294 */
1295unsigned char *skb_put(struct sk_buff *skb, unsigned int len)
1296{
1297 unsigned char *tmp = skb_tail_pointer(skb);
1298 SKB_LINEAR_ASSERT(skb);
1299 skb->tail += len;
1300 skb->len += len;
1301 if (unlikely(skb->tail > skb->end))
1302 skb_over_panic(skb, len, __builtin_return_address(0));
1303 return tmp;
1304}
1305EXPORT_SYMBOL(skb_put);
1306
Ilpo Järvinen6be8ac22008-03-27 17:47:24 -07001307/**
Ilpo Järvinenc2aa2702008-03-27 17:52:40 -07001308 * skb_push - add data to the start of a buffer
1309 * @skb: buffer to use
1310 * @len: amount of data to add
1311 *
1312 * This function extends the used data area of the buffer at the buffer
1313 * start. If this would exceed the total buffer headroom the kernel will
1314 * panic. A pointer to the first byte of the extra data is returned.
1315 */
1316unsigned char *skb_push(struct sk_buff *skb, unsigned int len)
1317{
1318 skb->data -= len;
1319 skb->len += len;
1320 if (unlikely(skb->data<skb->head))
1321 skb_under_panic(skb, len, __builtin_return_address(0));
1322 return skb->data;
1323}
1324EXPORT_SYMBOL(skb_push);
1325
1326/**
Ilpo Järvinen6be8ac22008-03-27 17:47:24 -07001327 * skb_pull - remove data from the start of a buffer
1328 * @skb: buffer to use
1329 * @len: amount of data to remove
1330 *
1331 * This function removes data from the start of a buffer, returning
1332 * the memory to the headroom. A pointer to the next data in the buffer
1333 * is returned. Once the data has been pulled future pushes will overwrite
1334 * the old data.
1335 */
1336unsigned char *skb_pull(struct sk_buff *skb, unsigned int len)
1337{
David S. Miller47d29642010-05-02 02:21:44 -07001338 return skb_pull_inline(skb, len);
Ilpo Järvinen6be8ac22008-03-27 17:47:24 -07001339}
1340EXPORT_SYMBOL(skb_pull);
1341
Ilpo Järvinen419ae742008-03-27 17:54:01 -07001342/**
1343 * skb_trim - remove end from a buffer
1344 * @skb: buffer to alter
1345 * @len: new length
1346 *
1347 * Cut the length of a buffer down by removing data from the tail. If
1348 * the buffer is already under the length specified it is not modified.
1349 * The skb must be linear.
1350 */
1351void skb_trim(struct sk_buff *skb, unsigned int len)
1352{
1353 if (skb->len > len)
1354 __skb_trim(skb, len);
1355}
1356EXPORT_SYMBOL(skb_trim);
1357
Herbert Xu3cc0e872006-06-09 16:13:38 -07001358/* Trims skb to length len. It can change skb pointers.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001359 */
1360
Herbert Xu3cc0e872006-06-09 16:13:38 -07001361int ___pskb_trim(struct sk_buff *skb, unsigned int len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001362{
Herbert Xu27b437c2006-07-13 19:26:39 -07001363 struct sk_buff **fragp;
1364 struct sk_buff *frag;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001365 int offset = skb_headlen(skb);
1366 int nfrags = skb_shinfo(skb)->nr_frags;
1367 int i;
Herbert Xu27b437c2006-07-13 19:26:39 -07001368 int err;
1369
1370 if (skb_cloned(skb) &&
1371 unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC))))
1372 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001373
Herbert Xuf4d26fb2006-07-30 20:20:28 -07001374 i = 0;
1375 if (offset >= len)
1376 goto drop_pages;
1377
1378 for (; i < nfrags; i++) {
Eric Dumazet9e903e02011-10-18 21:00:24 +00001379 int end = offset + skb_frag_size(&skb_shinfo(skb)->frags[i]);
Herbert Xu27b437c2006-07-13 19:26:39 -07001380
1381 if (end < len) {
1382 offset = end;
1383 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001384 }
Herbert Xu27b437c2006-07-13 19:26:39 -07001385
Eric Dumazet9e903e02011-10-18 21:00:24 +00001386 skb_frag_size_set(&skb_shinfo(skb)->frags[i++], len - offset);
Herbert Xu27b437c2006-07-13 19:26:39 -07001387
Herbert Xuf4d26fb2006-07-30 20:20:28 -07001388drop_pages:
Herbert Xu27b437c2006-07-13 19:26:39 -07001389 skb_shinfo(skb)->nr_frags = i;
1390
1391 for (; i < nfrags; i++)
Ian Campbellea2ab692011-08-22 23:44:58 +00001392 skb_frag_unref(skb, i);
Herbert Xu27b437c2006-07-13 19:26:39 -07001393
David S. Miller21dc3302010-08-23 00:13:46 -07001394 if (skb_has_frag_list(skb))
Herbert Xu27b437c2006-07-13 19:26:39 -07001395 skb_drop_fraglist(skb);
Herbert Xuf4d26fb2006-07-30 20:20:28 -07001396 goto done;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001397 }
1398
Herbert Xu27b437c2006-07-13 19:26:39 -07001399 for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp);
1400 fragp = &frag->next) {
1401 int end = offset + frag->len;
1402
1403 if (skb_shared(frag)) {
1404 struct sk_buff *nfrag;
1405
1406 nfrag = skb_clone(frag, GFP_ATOMIC);
1407 if (unlikely(!nfrag))
1408 return -ENOMEM;
1409
1410 nfrag->next = frag->next;
Eric Dumazet85bb2a62012-04-19 02:24:53 +00001411 consume_skb(frag);
Herbert Xu27b437c2006-07-13 19:26:39 -07001412 frag = nfrag;
1413 *fragp = frag;
1414 }
1415
1416 if (end < len) {
1417 offset = end;
1418 continue;
1419 }
1420
1421 if (end > len &&
1422 unlikely((err = pskb_trim(frag, len - offset))))
1423 return err;
1424
1425 if (frag->next)
1426 skb_drop_list(&frag->next);
1427 break;
1428 }
1429
Herbert Xuf4d26fb2006-07-30 20:20:28 -07001430done:
Herbert Xu27b437c2006-07-13 19:26:39 -07001431 if (len > skb_headlen(skb)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001432 skb->data_len -= skb->len - len;
1433 skb->len = len;
1434 } else {
Herbert Xu27b437c2006-07-13 19:26:39 -07001435 skb->len = len;
1436 skb->data_len = 0;
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -07001437 skb_set_tail_pointer(skb, len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001438 }
1439
1440 return 0;
1441}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08001442EXPORT_SYMBOL(___pskb_trim);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001443
1444/**
1445 * __pskb_pull_tail - advance tail of skb header
1446 * @skb: buffer to reallocate
1447 * @delta: number of bytes to advance tail
1448 *
1449 * The function makes a sense only on a fragmented &sk_buff,
1450 * it expands header moving its tail forward and copying necessary
1451 * data from fragmented part.
1452 *
1453 * &sk_buff MUST have reference count of 1.
1454 *
1455 * Returns %NULL (and &sk_buff does not change) if pull failed
1456 * or value of new tail of skb in the case of success.
1457 *
1458 * All the pointers pointing into skb header may change and must be
1459 * reloaded after call to this function.
1460 */
1461
1462/* Moves tail of skb head forward, copying data from fragmented part,
1463 * when it is necessary.
1464 * 1. It may fail due to malloc failure.
1465 * 2. It may change skb pointers.
1466 *
1467 * It is pretty complicated. Luckily, it is called only in exceptional cases.
1468 */
1469unsigned char *__pskb_pull_tail(struct sk_buff *skb, int delta)
1470{
1471 /* If skb has not enough free space at tail, get new one
1472 * plus 128 bytes for future expansions. If we have enough
1473 * room at tail, reallocate without expansion only if skb is cloned.
1474 */
Arnaldo Carvalho de Melo4305b542007-04-19 20:43:29 -07001475 int i, k, eat = (skb->tail + delta) - skb->end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001476
1477 if (eat > 0 || skb_cloned(skb)) {
1478 if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0,
1479 GFP_ATOMIC))
1480 return NULL;
1481 }
1482
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -07001483 if (skb_copy_bits(skb, skb_headlen(skb), skb_tail_pointer(skb), delta))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001484 BUG();
1485
1486 /* Optimization: no fragments, no reasons to preestimate
1487 * size of pulled pages. Superb.
1488 */
David S. Miller21dc3302010-08-23 00:13:46 -07001489 if (!skb_has_frag_list(skb))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001490 goto pull_pages;
1491
1492 /* Estimate size of pulled pages. */
1493 eat = delta;
1494 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
Eric Dumazet9e903e02011-10-18 21:00:24 +00001495 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
1496
1497 if (size >= eat)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001498 goto pull_pages;
Eric Dumazet9e903e02011-10-18 21:00:24 +00001499 eat -= size;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001500 }
1501
1502 /* If we need update frag list, we are in troubles.
1503 * Certainly, it possible to add an offset to skb data,
1504 * but taking into account that pulling is expected to
1505 * be very rare operation, it is worth to fight against
1506 * further bloating skb head and crucify ourselves here instead.
1507 * Pure masohism, indeed. 8)8)
1508 */
1509 if (eat) {
1510 struct sk_buff *list = skb_shinfo(skb)->frag_list;
1511 struct sk_buff *clone = NULL;
1512 struct sk_buff *insp = NULL;
1513
1514 do {
Kris Katterjohn09a62662006-01-08 22:24:28 -08001515 BUG_ON(!list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001516
1517 if (list->len <= eat) {
1518 /* Eaten as whole. */
1519 eat -= list->len;
1520 list = list->next;
1521 insp = list;
1522 } else {
1523 /* Eaten partially. */
1524
1525 if (skb_shared(list)) {
1526 /* Sucks! We need to fork list. :-( */
1527 clone = skb_clone(list, GFP_ATOMIC);
1528 if (!clone)
1529 return NULL;
1530 insp = list->next;
1531 list = clone;
1532 } else {
1533 /* This may be pulled without
1534 * problems. */
1535 insp = list;
1536 }
1537 if (!pskb_pull(list, eat)) {
Wei Yongjunf3fbbe02009-02-25 00:37:32 +00001538 kfree_skb(clone);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001539 return NULL;
1540 }
1541 break;
1542 }
1543 } while (eat);
1544
1545 /* Free pulled out fragments. */
1546 while ((list = skb_shinfo(skb)->frag_list) != insp) {
1547 skb_shinfo(skb)->frag_list = list->next;
1548 kfree_skb(list);
1549 }
1550 /* And insert new clone at head. */
1551 if (clone) {
1552 clone->next = list;
1553 skb_shinfo(skb)->frag_list = clone;
1554 }
1555 }
1556 /* Success! Now we may commit changes to skb data. */
1557
1558pull_pages:
1559 eat = delta;
1560 k = 0;
1561 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
Eric Dumazet9e903e02011-10-18 21:00:24 +00001562 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
1563
1564 if (size <= eat) {
Ian Campbellea2ab692011-08-22 23:44:58 +00001565 skb_frag_unref(skb, i);
Eric Dumazet9e903e02011-10-18 21:00:24 +00001566 eat -= size;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001567 } else {
1568 skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i];
1569 if (eat) {
1570 skb_shinfo(skb)->frags[k].page_offset += eat;
Eric Dumazet9e903e02011-10-18 21:00:24 +00001571 skb_frag_size_sub(&skb_shinfo(skb)->frags[k], eat);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001572 eat = 0;
1573 }
1574 k++;
1575 }
1576 }
1577 skb_shinfo(skb)->nr_frags = k;
1578
1579 skb->tail += delta;
1580 skb->data_len -= delta;
1581
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -07001582 return skb_tail_pointer(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001583}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08001584EXPORT_SYMBOL(__pskb_pull_tail);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001585
Eric Dumazet22019b12011-07-29 18:37:31 +00001586/**
1587 * skb_copy_bits - copy bits from skb to kernel buffer
1588 * @skb: source skb
1589 * @offset: offset in source
1590 * @to: destination buffer
1591 * @len: number of bytes to copy
1592 *
1593 * Copy the specified number of bytes from the source skb to the
1594 * destination buffer.
1595 *
1596 * CAUTION ! :
1597 * If its prototype is ever changed,
1598 * check arch/{*}/net/{*}.S files,
1599 * since it is called from BPF assembly code.
1600 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001601int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len)
1602{
David S. Miller1a028e52007-04-27 15:21:23 -07001603 int start = skb_headlen(skb);
David S. Millerfbb398a2009-06-09 00:18:59 -07001604 struct sk_buff *frag_iter;
1605 int i, copy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001606
1607 if (offset > (int)skb->len - len)
1608 goto fault;
1609
1610 /* Copy header. */
David S. Miller1a028e52007-04-27 15:21:23 -07001611 if ((copy = start - offset) > 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001612 if (copy > len)
1613 copy = len;
Arnaldo Carvalho de Melod626f622007-03-27 18:55:52 -03001614 skb_copy_from_linear_data_offset(skb, offset, to, copy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001615 if ((len -= copy) == 0)
1616 return 0;
1617 offset += copy;
1618 to += copy;
1619 }
1620
1621 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
David S. Miller1a028e52007-04-27 15:21:23 -07001622 int end;
Eric Dumazet51c56b02012-04-05 11:35:15 +02001623 skb_frag_t *f = &skb_shinfo(skb)->frags[i];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001624
Ilpo Järvinen547b7922008-07-25 21:43:18 -07001625 WARN_ON(start > offset + len);
David S. Miller1a028e52007-04-27 15:21:23 -07001626
Eric Dumazet51c56b02012-04-05 11:35:15 +02001627 end = start + skb_frag_size(f);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001628 if ((copy = end - offset) > 0) {
1629 u8 *vaddr;
1630
1631 if (copy > len)
1632 copy = len;
1633
Eric Dumazet51c56b02012-04-05 11:35:15 +02001634 vaddr = kmap_atomic(skb_frag_page(f));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001635 memcpy(to,
Eric Dumazet51c56b02012-04-05 11:35:15 +02001636 vaddr + f->page_offset + offset - start,
1637 copy);
1638 kunmap_atomic(vaddr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001639
1640 if ((len -= copy) == 0)
1641 return 0;
1642 offset += copy;
1643 to += copy;
1644 }
David S. Miller1a028e52007-04-27 15:21:23 -07001645 start = end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001646 }
1647
David S. Millerfbb398a2009-06-09 00:18:59 -07001648 skb_walk_frags(skb, frag_iter) {
1649 int end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001650
David S. Millerfbb398a2009-06-09 00:18:59 -07001651 WARN_ON(start > offset + len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001652
David S. Millerfbb398a2009-06-09 00:18:59 -07001653 end = start + frag_iter->len;
1654 if ((copy = end - offset) > 0) {
1655 if (copy > len)
1656 copy = len;
1657 if (skb_copy_bits(frag_iter, offset - start, to, copy))
1658 goto fault;
1659 if ((len -= copy) == 0)
1660 return 0;
1661 offset += copy;
1662 to += copy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001663 }
David S. Millerfbb398a2009-06-09 00:18:59 -07001664 start = end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001665 }
Shirley Maa6686f22011-07-06 12:22:12 +00001666
Linus Torvalds1da177e2005-04-16 15:20:36 -07001667 if (!len)
1668 return 0;
1669
1670fault:
1671 return -EFAULT;
1672}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08001673EXPORT_SYMBOL(skb_copy_bits);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001674
Jens Axboe9c55e012007-11-06 23:30:13 -08001675/*
1676 * Callback from splice_to_pipe(), if we need to release some pages
1677 * at the end of the spd in case we error'ed out in filling the pipe.
1678 */
1679static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i)
1680{
Jarek Poplawski8b9d3722009-01-19 17:03:56 -08001681 put_page(spd->pages[i]);
1682}
Jens Axboe9c55e012007-11-06 23:30:13 -08001683
David S. Millera108d5f2012-04-23 23:06:11 -04001684static struct page *linear_to_page(struct page *page, unsigned int *len,
1685 unsigned int *offset,
Eric Dumazet18aafc62013-01-11 14:46:37 +00001686 struct sock *sk)
Jarek Poplawski8b9d3722009-01-19 17:03:56 -08001687{
Eric Dumazet5640f762012-09-23 23:04:42 +00001688 struct page_frag *pfrag = sk_page_frag(sk);
Jarek Poplawski8b9d3722009-01-19 17:03:56 -08001689
Eric Dumazet5640f762012-09-23 23:04:42 +00001690 if (!sk_page_frag_refill(sk, pfrag))
1691 return NULL;
Jarek Poplawski4fb66992009-02-01 00:41:42 -08001692
Eric Dumazet5640f762012-09-23 23:04:42 +00001693 *len = min_t(unsigned int, *len, pfrag->size - pfrag->offset);
Jarek Poplawski4fb66992009-02-01 00:41:42 -08001694
Eric Dumazet5640f762012-09-23 23:04:42 +00001695 memcpy(page_address(pfrag->page) + pfrag->offset,
1696 page_address(page) + *offset, *len);
1697 *offset = pfrag->offset;
1698 pfrag->offset += *len;
Jarek Poplawski4fb66992009-02-01 00:41:42 -08001699
Eric Dumazet5640f762012-09-23 23:04:42 +00001700 return pfrag->page;
Jens Axboe9c55e012007-11-06 23:30:13 -08001701}
1702
Eric Dumazet41c73a02012-04-22 12:26:16 +00001703static bool spd_can_coalesce(const struct splice_pipe_desc *spd,
1704 struct page *page,
1705 unsigned int offset)
1706{
1707 return spd->nr_pages &&
1708 spd->pages[spd->nr_pages - 1] == page &&
1709 (spd->partial[spd->nr_pages - 1].offset +
1710 spd->partial[spd->nr_pages - 1].len == offset);
1711}
1712
Jens Axboe9c55e012007-11-06 23:30:13 -08001713/*
1714 * Fill page/offset/length into spd, if it can hold more pages.
1715 */
David S. Millera108d5f2012-04-23 23:06:11 -04001716static bool spd_fill_page(struct splice_pipe_desc *spd,
1717 struct pipe_inode_info *pipe, struct page *page,
1718 unsigned int *len, unsigned int offset,
Eric Dumazet18aafc62013-01-11 14:46:37 +00001719 bool linear,
David S. Millera108d5f2012-04-23 23:06:11 -04001720 struct sock *sk)
Jens Axboe9c55e012007-11-06 23:30:13 -08001721{
Eric Dumazet41c73a02012-04-22 12:26:16 +00001722 if (unlikely(spd->nr_pages == MAX_SKB_FRAGS))
David S. Millera108d5f2012-04-23 23:06:11 -04001723 return true;
Jens Axboe9c55e012007-11-06 23:30:13 -08001724
Jarek Poplawski8b9d3722009-01-19 17:03:56 -08001725 if (linear) {
Eric Dumazet18aafc62013-01-11 14:46:37 +00001726 page = linear_to_page(page, len, &offset, sk);
Jarek Poplawski8b9d3722009-01-19 17:03:56 -08001727 if (!page)
David S. Millera108d5f2012-04-23 23:06:11 -04001728 return true;
Eric Dumazet41c73a02012-04-22 12:26:16 +00001729 }
1730 if (spd_can_coalesce(spd, page, offset)) {
1731 spd->partial[spd->nr_pages - 1].len += *len;
David S. Millera108d5f2012-04-23 23:06:11 -04001732 return false;
Eric Dumazet41c73a02012-04-22 12:26:16 +00001733 }
1734 get_page(page);
Jens Axboe9c55e012007-11-06 23:30:13 -08001735 spd->pages[spd->nr_pages] = page;
Jarek Poplawski4fb66992009-02-01 00:41:42 -08001736 spd->partial[spd->nr_pages].len = *len;
Jens Axboe9c55e012007-11-06 23:30:13 -08001737 spd->partial[spd->nr_pages].offset = offset;
Jens Axboe9c55e012007-11-06 23:30:13 -08001738 spd->nr_pages++;
Jarek Poplawski8b9d3722009-01-19 17:03:56 -08001739
David S. Millera108d5f2012-04-23 23:06:11 -04001740 return false;
Jens Axboe9c55e012007-11-06 23:30:13 -08001741}
1742
David S. Millera108d5f2012-04-23 23:06:11 -04001743static bool __splice_segment(struct page *page, unsigned int poff,
1744 unsigned int plen, unsigned int *off,
Eric Dumazet18aafc62013-01-11 14:46:37 +00001745 unsigned int *len,
Eric Dumazetd7ccf7c2012-04-23 23:35:04 -04001746 struct splice_pipe_desc *spd, bool linear,
David S. Millera108d5f2012-04-23 23:06:11 -04001747 struct sock *sk,
1748 struct pipe_inode_info *pipe)
Octavian Purdila2870c432008-07-15 00:49:11 -07001749{
1750 if (!*len)
David S. Millera108d5f2012-04-23 23:06:11 -04001751 return true;
Octavian Purdila2870c432008-07-15 00:49:11 -07001752
1753 /* skip this segment if already processed */
1754 if (*off >= plen) {
1755 *off -= plen;
David S. Millera108d5f2012-04-23 23:06:11 -04001756 return false;
Octavian Purdiladb43a282008-06-27 17:27:21 -07001757 }
Jens Axboe9c55e012007-11-06 23:30:13 -08001758
Octavian Purdila2870c432008-07-15 00:49:11 -07001759 /* ignore any bits we already processed */
Eric Dumazet9ca1b222013-01-05 21:31:18 +00001760 poff += *off;
1761 plen -= *off;
1762 *off = 0;
Octavian Purdila2870c432008-07-15 00:49:11 -07001763
Eric Dumazet18aafc62013-01-11 14:46:37 +00001764 do {
1765 unsigned int flen = min(*len, plen);
Octavian Purdila2870c432008-07-15 00:49:11 -07001766
Eric Dumazet18aafc62013-01-11 14:46:37 +00001767 if (spd_fill_page(spd, pipe, page, &flen, poff,
1768 linear, sk))
1769 return true;
1770 poff += flen;
1771 plen -= flen;
1772 *len -= flen;
1773 } while (*len && plen);
Octavian Purdila2870c432008-07-15 00:49:11 -07001774
David S. Millera108d5f2012-04-23 23:06:11 -04001775 return false;
Octavian Purdila2870c432008-07-15 00:49:11 -07001776}
1777
1778/*
David S. Millera108d5f2012-04-23 23:06:11 -04001779 * Map linear and fragment data from the skb to spd. It reports true if the
Octavian Purdila2870c432008-07-15 00:49:11 -07001780 * pipe is full or if we already spliced the requested length.
1781 */
David S. Millera108d5f2012-04-23 23:06:11 -04001782static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe,
1783 unsigned int *offset, unsigned int *len,
1784 struct splice_pipe_desc *spd, struct sock *sk)
Octavian Purdila2870c432008-07-15 00:49:11 -07001785{
1786 int seg;
1787
Eric Dumazet1d0c0b32012-04-27 02:10:03 +00001788 /* map the linear part :
Alexander Duyck2996d312012-05-02 18:18:42 +00001789 * If skb->head_frag is set, this 'linear' part is backed by a
1790 * fragment, and if the head is not shared with any clones then
1791 * we can avoid a copy since we own the head portion of this page.
Jens Axboe9c55e012007-11-06 23:30:13 -08001792 */
Octavian Purdila2870c432008-07-15 00:49:11 -07001793 if (__splice_segment(virt_to_page(skb->data),
1794 (unsigned long) skb->data & (PAGE_SIZE - 1),
1795 skb_headlen(skb),
Eric Dumazet18aafc62013-01-11 14:46:37 +00001796 offset, len, spd,
Alexander Duyck3a7c1ee42012-05-03 01:09:42 +00001797 skb_head_is_locked(skb),
Eric Dumazet1d0c0b32012-04-27 02:10:03 +00001798 sk, pipe))
David S. Millera108d5f2012-04-23 23:06:11 -04001799 return true;
Jens Axboe9c55e012007-11-06 23:30:13 -08001800
1801 /*
1802 * then map the fragments
1803 */
Jens Axboe9c55e012007-11-06 23:30:13 -08001804 for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) {
1805 const skb_frag_t *f = &skb_shinfo(skb)->frags[seg];
1806
Ian Campbellea2ab692011-08-22 23:44:58 +00001807 if (__splice_segment(skb_frag_page(f),
Eric Dumazet9e903e02011-10-18 21:00:24 +00001808 f->page_offset, skb_frag_size(f),
Eric Dumazet18aafc62013-01-11 14:46:37 +00001809 offset, len, spd, false, sk, pipe))
David S. Millera108d5f2012-04-23 23:06:11 -04001810 return true;
Jens Axboe9c55e012007-11-06 23:30:13 -08001811 }
1812
David S. Millera108d5f2012-04-23 23:06:11 -04001813 return false;
Jens Axboe9c55e012007-11-06 23:30:13 -08001814}
1815
1816/*
1817 * Map data from the skb to a pipe. Should handle both the linear part,
1818 * the fragments, and the frag list. It does NOT handle frag lists within
1819 * the frag list, if such a thing exists. We'd probably need to recurse to
1820 * handle that cleanly.
1821 */
Jarek Poplawski8b9d3722009-01-19 17:03:56 -08001822int skb_splice_bits(struct sk_buff *skb, unsigned int offset,
Jens Axboe9c55e012007-11-06 23:30:13 -08001823 struct pipe_inode_info *pipe, unsigned int tlen,
1824 unsigned int flags)
1825{
Eric Dumazet41c73a02012-04-22 12:26:16 +00001826 struct partial_page partial[MAX_SKB_FRAGS];
1827 struct page *pages[MAX_SKB_FRAGS];
Jens Axboe9c55e012007-11-06 23:30:13 -08001828 struct splice_pipe_desc spd = {
1829 .pages = pages,
1830 .partial = partial,
Eric Dumazet047fe362012-06-12 15:24:40 +02001831 .nr_pages_max = MAX_SKB_FRAGS,
Jens Axboe9c55e012007-11-06 23:30:13 -08001832 .flags = flags,
1833 .ops = &sock_pipe_buf_ops,
1834 .spd_release = sock_spd_release,
1835 };
David S. Millerfbb398a2009-06-09 00:18:59 -07001836 struct sk_buff *frag_iter;
Jarek Poplawski7a67e562009-04-30 05:41:19 -07001837 struct sock *sk = skb->sk;
Jens Axboe35f3d142010-05-20 10:43:18 +02001838 int ret = 0;
1839
Jens Axboe9c55e012007-11-06 23:30:13 -08001840 /*
1841 * __skb_splice_bits() only fails if the output has no room left,
1842 * so no point in going over the frag_list for the error case.
1843 */
Jens Axboe35f3d142010-05-20 10:43:18 +02001844 if (__skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk))
Jens Axboe9c55e012007-11-06 23:30:13 -08001845 goto done;
1846 else if (!tlen)
1847 goto done;
1848
1849 /*
1850 * now see if we have a frag_list to map
1851 */
David S. Millerfbb398a2009-06-09 00:18:59 -07001852 skb_walk_frags(skb, frag_iter) {
1853 if (!tlen)
1854 break;
Jens Axboe35f3d142010-05-20 10:43:18 +02001855 if (__skb_splice_bits(frag_iter, pipe, &offset, &tlen, &spd, sk))
David S. Millerfbb398a2009-06-09 00:18:59 -07001856 break;
Jens Axboe9c55e012007-11-06 23:30:13 -08001857 }
1858
1859done:
Jens Axboe9c55e012007-11-06 23:30:13 -08001860 if (spd.nr_pages) {
Jens Axboe9c55e012007-11-06 23:30:13 -08001861 /*
1862 * Drop the socket lock, otherwise we have reverse
1863 * locking dependencies between sk_lock and i_mutex
1864 * here as compared to sendfile(). We enter here
1865 * with the socket lock held, and splice_to_pipe() will
1866 * grab the pipe inode lock. For sendfile() emulation,
1867 * we call into ->sendpage() with the i_mutex lock held
1868 * and networking will grab the socket lock.
1869 */
Octavian Purdila293ad602008-06-04 15:45:58 -07001870 release_sock(sk);
Jens Axboe9c55e012007-11-06 23:30:13 -08001871 ret = splice_to_pipe(pipe, &spd);
Octavian Purdila293ad602008-06-04 15:45:58 -07001872 lock_sock(sk);
Jens Axboe9c55e012007-11-06 23:30:13 -08001873 }
1874
Jens Axboe35f3d142010-05-20 10:43:18 +02001875 return ret;
Jens Axboe9c55e012007-11-06 23:30:13 -08001876}
1877
Herbert Xu357b40a2005-04-19 22:30:14 -07001878/**
1879 * skb_store_bits - store bits from kernel buffer to skb
1880 * @skb: destination buffer
1881 * @offset: offset in destination
1882 * @from: source buffer
1883 * @len: number of bytes to copy
1884 *
1885 * Copy the specified number of bytes from the source buffer to the
1886 * destination skb. This function handles all the messy bits of
1887 * traversing fragment lists and such.
1888 */
1889
Stephen Hemminger0c6fcc82007-04-20 16:40:01 -07001890int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len)
Herbert Xu357b40a2005-04-19 22:30:14 -07001891{
David S. Miller1a028e52007-04-27 15:21:23 -07001892 int start = skb_headlen(skb);
David S. Millerfbb398a2009-06-09 00:18:59 -07001893 struct sk_buff *frag_iter;
1894 int i, copy;
Herbert Xu357b40a2005-04-19 22:30:14 -07001895
1896 if (offset > (int)skb->len - len)
1897 goto fault;
1898
David S. Miller1a028e52007-04-27 15:21:23 -07001899 if ((copy = start - offset) > 0) {
Herbert Xu357b40a2005-04-19 22:30:14 -07001900 if (copy > len)
1901 copy = len;
Arnaldo Carvalho de Melo27d7ff42007-03-31 11:55:19 -03001902 skb_copy_to_linear_data_offset(skb, offset, from, copy);
Herbert Xu357b40a2005-04-19 22:30:14 -07001903 if ((len -= copy) == 0)
1904 return 0;
1905 offset += copy;
1906 from += copy;
1907 }
1908
1909 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1910 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
David S. Miller1a028e52007-04-27 15:21:23 -07001911 int end;
Herbert Xu357b40a2005-04-19 22:30:14 -07001912
Ilpo Järvinen547b7922008-07-25 21:43:18 -07001913 WARN_ON(start > offset + len);
David S. Miller1a028e52007-04-27 15:21:23 -07001914
Eric Dumazet9e903e02011-10-18 21:00:24 +00001915 end = start + skb_frag_size(frag);
Herbert Xu357b40a2005-04-19 22:30:14 -07001916 if ((copy = end - offset) > 0) {
1917 u8 *vaddr;
1918
1919 if (copy > len)
1920 copy = len;
1921
Eric Dumazet51c56b02012-04-05 11:35:15 +02001922 vaddr = kmap_atomic(skb_frag_page(frag));
David S. Miller1a028e52007-04-27 15:21:23 -07001923 memcpy(vaddr + frag->page_offset + offset - start,
1924 from, copy);
Eric Dumazet51c56b02012-04-05 11:35:15 +02001925 kunmap_atomic(vaddr);
Herbert Xu357b40a2005-04-19 22:30:14 -07001926
1927 if ((len -= copy) == 0)
1928 return 0;
1929 offset += copy;
1930 from += copy;
1931 }
David S. Miller1a028e52007-04-27 15:21:23 -07001932 start = end;
Herbert Xu357b40a2005-04-19 22:30:14 -07001933 }
1934
David S. Millerfbb398a2009-06-09 00:18:59 -07001935 skb_walk_frags(skb, frag_iter) {
1936 int end;
Herbert Xu357b40a2005-04-19 22:30:14 -07001937
David S. Millerfbb398a2009-06-09 00:18:59 -07001938 WARN_ON(start > offset + len);
Herbert Xu357b40a2005-04-19 22:30:14 -07001939
David S. Millerfbb398a2009-06-09 00:18:59 -07001940 end = start + frag_iter->len;
1941 if ((copy = end - offset) > 0) {
1942 if (copy > len)
1943 copy = len;
1944 if (skb_store_bits(frag_iter, offset - start,
1945 from, copy))
1946 goto fault;
1947 if ((len -= copy) == 0)
1948 return 0;
1949 offset += copy;
1950 from += copy;
Herbert Xu357b40a2005-04-19 22:30:14 -07001951 }
David S. Millerfbb398a2009-06-09 00:18:59 -07001952 start = end;
Herbert Xu357b40a2005-04-19 22:30:14 -07001953 }
1954 if (!len)
1955 return 0;
1956
1957fault:
1958 return -EFAULT;
1959}
Herbert Xu357b40a2005-04-19 22:30:14 -07001960EXPORT_SYMBOL(skb_store_bits);
1961
Linus Torvalds1da177e2005-04-16 15:20:36 -07001962/* Checksum skb data. */
Daniel Borkmann2817a332013-10-30 11:50:51 +01001963__wsum __skb_checksum(const struct sk_buff *skb, int offset, int len,
1964 __wsum csum, const struct skb_checksum_ops *ops)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001965{
David S. Miller1a028e52007-04-27 15:21:23 -07001966 int start = skb_headlen(skb);
1967 int i, copy = start - offset;
David S. Millerfbb398a2009-06-09 00:18:59 -07001968 struct sk_buff *frag_iter;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001969 int pos = 0;
1970
1971 /* Checksum header. */
1972 if (copy > 0) {
1973 if (copy > len)
1974 copy = len;
Daniel Borkmann2817a332013-10-30 11:50:51 +01001975 csum = ops->update(skb->data + offset, copy, csum);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001976 if ((len -= copy) == 0)
1977 return csum;
1978 offset += copy;
1979 pos = copy;
1980 }
1981
1982 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
David S. Miller1a028e52007-04-27 15:21:23 -07001983 int end;
Eric Dumazet51c56b02012-04-05 11:35:15 +02001984 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001985
Ilpo Järvinen547b7922008-07-25 21:43:18 -07001986 WARN_ON(start > offset + len);
David S. Miller1a028e52007-04-27 15:21:23 -07001987
Eric Dumazet51c56b02012-04-05 11:35:15 +02001988 end = start + skb_frag_size(frag);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001989 if ((copy = end - offset) > 0) {
Al Viro44bb9362006-11-14 21:36:14 -08001990 __wsum csum2;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001991 u8 *vaddr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001992
1993 if (copy > len)
1994 copy = len;
Eric Dumazet51c56b02012-04-05 11:35:15 +02001995 vaddr = kmap_atomic(skb_frag_page(frag));
Daniel Borkmann2817a332013-10-30 11:50:51 +01001996 csum2 = ops->update(vaddr + frag->page_offset +
1997 offset - start, copy, 0);
Eric Dumazet51c56b02012-04-05 11:35:15 +02001998 kunmap_atomic(vaddr);
Daniel Borkmann2817a332013-10-30 11:50:51 +01001999 csum = ops->combine(csum, csum2, pos, copy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002000 if (!(len -= copy))
2001 return csum;
2002 offset += copy;
2003 pos += copy;
2004 }
David S. Miller1a028e52007-04-27 15:21:23 -07002005 start = end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002006 }
2007
David S. Millerfbb398a2009-06-09 00:18:59 -07002008 skb_walk_frags(skb, frag_iter) {
2009 int end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002010
David S. Millerfbb398a2009-06-09 00:18:59 -07002011 WARN_ON(start > offset + len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002012
David S. Millerfbb398a2009-06-09 00:18:59 -07002013 end = start + frag_iter->len;
2014 if ((copy = end - offset) > 0) {
2015 __wsum csum2;
2016 if (copy > len)
2017 copy = len;
Daniel Borkmann2817a332013-10-30 11:50:51 +01002018 csum2 = __skb_checksum(frag_iter, offset - start,
2019 copy, 0, ops);
2020 csum = ops->combine(csum, csum2, pos, copy);
David S. Millerfbb398a2009-06-09 00:18:59 -07002021 if ((len -= copy) == 0)
2022 return csum;
2023 offset += copy;
2024 pos += copy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002025 }
David S. Millerfbb398a2009-06-09 00:18:59 -07002026 start = end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002027 }
Kris Katterjohn09a62662006-01-08 22:24:28 -08002028 BUG_ON(len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002029
2030 return csum;
2031}
Daniel Borkmann2817a332013-10-30 11:50:51 +01002032EXPORT_SYMBOL(__skb_checksum);
2033
2034__wsum skb_checksum(const struct sk_buff *skb, int offset,
2035 int len, __wsum csum)
2036{
2037 const struct skb_checksum_ops ops = {
Daniel Borkmanncea80ea2013-11-04 17:10:25 +01002038 .update = csum_partial_ext,
Daniel Borkmann2817a332013-10-30 11:50:51 +01002039 .combine = csum_block_add_ext,
2040 };
2041
2042 return __skb_checksum(skb, offset, len, csum, &ops);
2043}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08002044EXPORT_SYMBOL(skb_checksum);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002045
2046/* Both of above in one bottle. */
2047
Al Viro81d77662006-11-14 21:37:33 -08002048__wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset,
2049 u8 *to, int len, __wsum csum)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002050{
David S. Miller1a028e52007-04-27 15:21:23 -07002051 int start = skb_headlen(skb);
2052 int i, copy = start - offset;
David S. Millerfbb398a2009-06-09 00:18:59 -07002053 struct sk_buff *frag_iter;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002054 int pos = 0;
2055
2056 /* Copy header. */
2057 if (copy > 0) {
2058 if (copy > len)
2059 copy = len;
2060 csum = csum_partial_copy_nocheck(skb->data + offset, to,
2061 copy, csum);
2062 if ((len -= copy) == 0)
2063 return csum;
2064 offset += copy;
2065 to += copy;
2066 pos = copy;
2067 }
2068
2069 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
David S. Miller1a028e52007-04-27 15:21:23 -07002070 int end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002071
Ilpo Järvinen547b7922008-07-25 21:43:18 -07002072 WARN_ON(start > offset + len);
David S. Miller1a028e52007-04-27 15:21:23 -07002073
Eric Dumazet9e903e02011-10-18 21:00:24 +00002074 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002075 if ((copy = end - offset) > 0) {
Al Viro50842052006-11-14 21:36:34 -08002076 __wsum csum2;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002077 u8 *vaddr;
2078 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2079
2080 if (copy > len)
2081 copy = len;
Eric Dumazet51c56b02012-04-05 11:35:15 +02002082 vaddr = kmap_atomic(skb_frag_page(frag));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002083 csum2 = csum_partial_copy_nocheck(vaddr +
David S. Miller1a028e52007-04-27 15:21:23 -07002084 frag->page_offset +
2085 offset - start, to,
2086 copy, 0);
Eric Dumazet51c56b02012-04-05 11:35:15 +02002087 kunmap_atomic(vaddr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002088 csum = csum_block_add(csum, csum2, pos);
2089 if (!(len -= copy))
2090 return csum;
2091 offset += copy;
2092 to += copy;
2093 pos += copy;
2094 }
David S. Miller1a028e52007-04-27 15:21:23 -07002095 start = end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002096 }
2097
David S. Millerfbb398a2009-06-09 00:18:59 -07002098 skb_walk_frags(skb, frag_iter) {
2099 __wsum csum2;
2100 int end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002101
David S. Millerfbb398a2009-06-09 00:18:59 -07002102 WARN_ON(start > offset + len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002103
David S. Millerfbb398a2009-06-09 00:18:59 -07002104 end = start + frag_iter->len;
2105 if ((copy = end - offset) > 0) {
2106 if (copy > len)
2107 copy = len;
2108 csum2 = skb_copy_and_csum_bits(frag_iter,
2109 offset - start,
2110 to, copy, 0);
2111 csum = csum_block_add(csum, csum2, pos);
2112 if ((len -= copy) == 0)
2113 return csum;
2114 offset += copy;
2115 to += copy;
2116 pos += copy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002117 }
David S. Millerfbb398a2009-06-09 00:18:59 -07002118 start = end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002119 }
Kris Katterjohn09a62662006-01-08 22:24:28 -08002120 BUG_ON(len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002121 return csum;
2122}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08002123EXPORT_SYMBOL(skb_copy_and_csum_bits);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002124
2125void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to)
2126{
Al Virod3bc23e2006-11-14 21:24:49 -08002127 __wsum csum;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002128 long csstart;
2129
Patrick McHardy84fa7932006-08-29 16:44:56 -07002130 if (skb->ip_summed == CHECKSUM_PARTIAL)
Michał Mirosław55508d62010-12-14 15:24:08 +00002131 csstart = skb_checksum_start_offset(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002132 else
2133 csstart = skb_headlen(skb);
2134
Kris Katterjohn09a62662006-01-08 22:24:28 -08002135 BUG_ON(csstart > skb_headlen(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002136
Arnaldo Carvalho de Melod626f622007-03-27 18:55:52 -03002137 skb_copy_from_linear_data(skb, to, csstart);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002138
2139 csum = 0;
2140 if (csstart != skb->len)
2141 csum = skb_copy_and_csum_bits(skb, csstart, to + csstart,
2142 skb->len - csstart, 0);
2143
Patrick McHardy84fa7932006-08-29 16:44:56 -07002144 if (skb->ip_summed == CHECKSUM_PARTIAL) {
Al Viroff1dcad2006-11-20 18:07:29 -08002145 long csstuff = csstart + skb->csum_offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002146
Al Virod3bc23e2006-11-14 21:24:49 -08002147 *((__sum16 *)(to + csstuff)) = csum_fold(csum);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002148 }
2149}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08002150EXPORT_SYMBOL(skb_copy_and_csum_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002151
2152/**
2153 * skb_dequeue - remove from the head of the queue
2154 * @list: list to dequeue from
2155 *
2156 * Remove the head of the list. The list lock is taken so the function
2157 * may be used safely with other locking list functions. The head item is
2158 * returned or %NULL if the list is empty.
2159 */
2160
2161struct sk_buff *skb_dequeue(struct sk_buff_head *list)
2162{
2163 unsigned long flags;
2164 struct sk_buff *result;
2165
2166 spin_lock_irqsave(&list->lock, flags);
2167 result = __skb_dequeue(list);
2168 spin_unlock_irqrestore(&list->lock, flags);
2169 return result;
2170}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08002171EXPORT_SYMBOL(skb_dequeue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002172
2173/**
2174 * skb_dequeue_tail - remove from the tail of the queue
2175 * @list: list to dequeue from
2176 *
2177 * Remove the tail of the list. The list lock is taken so the function
2178 * may be used safely with other locking list functions. The tail item is
2179 * returned or %NULL if the list is empty.
2180 */
2181struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list)
2182{
2183 unsigned long flags;
2184 struct sk_buff *result;
2185
2186 spin_lock_irqsave(&list->lock, flags);
2187 result = __skb_dequeue_tail(list);
2188 spin_unlock_irqrestore(&list->lock, flags);
2189 return result;
2190}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08002191EXPORT_SYMBOL(skb_dequeue_tail);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002192
2193/**
2194 * skb_queue_purge - empty a list
2195 * @list: list to empty
2196 *
2197 * Delete all buffers on an &sk_buff list. Each buffer is removed from
2198 * the list and one reference dropped. This function takes the list
2199 * lock and is atomic with respect to other list locking functions.
2200 */
2201void skb_queue_purge(struct sk_buff_head *list)
2202{
2203 struct sk_buff *skb;
2204 while ((skb = skb_dequeue(list)) != NULL)
2205 kfree_skb(skb);
2206}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08002207EXPORT_SYMBOL(skb_queue_purge);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002208
2209/**
2210 * skb_queue_head - queue a buffer at the list head
2211 * @list: list to use
2212 * @newsk: buffer to queue
2213 *
2214 * Queue a buffer at the start of the list. This function takes the
2215 * list lock and can be used safely with other locking &sk_buff functions
2216 * safely.
2217 *
2218 * A buffer cannot be placed on two lists at the same time.
2219 */
2220void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk)
2221{
2222 unsigned long flags;
2223
2224 spin_lock_irqsave(&list->lock, flags);
2225 __skb_queue_head(list, newsk);
2226 spin_unlock_irqrestore(&list->lock, flags);
2227}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08002228EXPORT_SYMBOL(skb_queue_head);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002229
2230/**
2231 * skb_queue_tail - queue a buffer at the list tail
2232 * @list: list to use
2233 * @newsk: buffer to queue
2234 *
2235 * Queue a buffer at the tail of the list. This function takes the
2236 * list lock and can be used safely with other locking &sk_buff functions
2237 * safely.
2238 *
2239 * A buffer cannot be placed on two lists at the same time.
2240 */
2241void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk)
2242{
2243 unsigned long flags;
2244
2245 spin_lock_irqsave(&list->lock, flags);
2246 __skb_queue_tail(list, newsk);
2247 spin_unlock_irqrestore(&list->lock, flags);
2248}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08002249EXPORT_SYMBOL(skb_queue_tail);
David S. Miller8728b832005-08-09 19:25:21 -07002250
Linus Torvalds1da177e2005-04-16 15:20:36 -07002251/**
2252 * skb_unlink - remove a buffer from a list
2253 * @skb: buffer to remove
David S. Miller8728b832005-08-09 19:25:21 -07002254 * @list: list to use
Linus Torvalds1da177e2005-04-16 15:20:36 -07002255 *
David S. Miller8728b832005-08-09 19:25:21 -07002256 * Remove a packet from a list. The list locks are taken and this
2257 * function is atomic with respect to other list locked calls
Linus Torvalds1da177e2005-04-16 15:20:36 -07002258 *
David S. Miller8728b832005-08-09 19:25:21 -07002259 * You must know what list the SKB is on.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002260 */
David S. Miller8728b832005-08-09 19:25:21 -07002261void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002262{
David S. Miller8728b832005-08-09 19:25:21 -07002263 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002264
David S. Miller8728b832005-08-09 19:25:21 -07002265 spin_lock_irqsave(&list->lock, flags);
2266 __skb_unlink(skb, list);
2267 spin_unlock_irqrestore(&list->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002268}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08002269EXPORT_SYMBOL(skb_unlink);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002270
Linus Torvalds1da177e2005-04-16 15:20:36 -07002271/**
2272 * skb_append - append a buffer
2273 * @old: buffer to insert after
2274 * @newsk: buffer to insert
David S. Miller8728b832005-08-09 19:25:21 -07002275 * @list: list to use
Linus Torvalds1da177e2005-04-16 15:20:36 -07002276 *
2277 * Place a packet after a given packet in a list. The list locks are taken
2278 * and this function is atomic with respect to other list locked calls.
2279 * A buffer cannot be placed on two lists at the same time.
2280 */
David S. Miller8728b832005-08-09 19:25:21 -07002281void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002282{
2283 unsigned long flags;
2284
David S. Miller8728b832005-08-09 19:25:21 -07002285 spin_lock_irqsave(&list->lock, flags);
Gerrit Renker7de6c032008-04-14 00:05:09 -07002286 __skb_queue_after(list, old, newsk);
David S. Miller8728b832005-08-09 19:25:21 -07002287 spin_unlock_irqrestore(&list->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002288}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08002289EXPORT_SYMBOL(skb_append);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002290
2291/**
2292 * skb_insert - insert a buffer
2293 * @old: buffer to insert before
2294 * @newsk: buffer to insert
David S. Miller8728b832005-08-09 19:25:21 -07002295 * @list: list to use
Linus Torvalds1da177e2005-04-16 15:20:36 -07002296 *
David S. Miller8728b832005-08-09 19:25:21 -07002297 * Place a packet before a given packet in a list. The list locks are
2298 * taken and this function is atomic with respect to other list locked
2299 * calls.
2300 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07002301 * A buffer cannot be placed on two lists at the same time.
2302 */
David S. Miller8728b832005-08-09 19:25:21 -07002303void skb_insert(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002304{
2305 unsigned long flags;
2306
David S. Miller8728b832005-08-09 19:25:21 -07002307 spin_lock_irqsave(&list->lock, flags);
2308 __skb_insert(newsk, old->prev, old, list);
2309 spin_unlock_irqrestore(&list->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002310}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08002311EXPORT_SYMBOL(skb_insert);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002312
Linus Torvalds1da177e2005-04-16 15:20:36 -07002313static inline void skb_split_inside_header(struct sk_buff *skb,
2314 struct sk_buff* skb1,
2315 const u32 len, const int pos)
2316{
2317 int i;
2318
Arnaldo Carvalho de Melod626f622007-03-27 18:55:52 -03002319 skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len),
2320 pos - len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002321 /* And move data appendix as is. */
2322 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
2323 skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i];
2324
2325 skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags;
2326 skb_shinfo(skb)->nr_frags = 0;
2327 skb1->data_len = skb->data_len;
2328 skb1->len += skb1->data_len;
2329 skb->data_len = 0;
2330 skb->len = len;
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -07002331 skb_set_tail_pointer(skb, len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002332}
2333
2334static inline void skb_split_no_header(struct sk_buff *skb,
2335 struct sk_buff* skb1,
2336 const u32 len, int pos)
2337{
2338 int i, k = 0;
2339 const int nfrags = skb_shinfo(skb)->nr_frags;
2340
2341 skb_shinfo(skb)->nr_frags = 0;
2342 skb1->len = skb1->data_len = skb->len - len;
2343 skb->len = len;
2344 skb->data_len = len - pos;
2345
2346 for (i = 0; i < nfrags; i++) {
Eric Dumazet9e903e02011-10-18 21:00:24 +00002347 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002348
2349 if (pos + size > len) {
2350 skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i];
2351
2352 if (pos < len) {
2353 /* Split frag.
2354 * We have two variants in this case:
2355 * 1. Move all the frag to the second
2356 * part, if it is possible. F.e.
2357 * this approach is mandatory for TUX,
2358 * where splitting is expensive.
2359 * 2. Split is accurately. We make this.
2360 */
Ian Campbellea2ab692011-08-22 23:44:58 +00002361 skb_frag_ref(skb, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002362 skb_shinfo(skb1)->frags[0].page_offset += len - pos;
Eric Dumazet9e903e02011-10-18 21:00:24 +00002363 skb_frag_size_sub(&skb_shinfo(skb1)->frags[0], len - pos);
2364 skb_frag_size_set(&skb_shinfo(skb)->frags[i], len - pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002365 skb_shinfo(skb)->nr_frags++;
2366 }
2367 k++;
2368 } else
2369 skb_shinfo(skb)->nr_frags++;
2370 pos += size;
2371 }
2372 skb_shinfo(skb1)->nr_frags = k;
2373}
2374
2375/**
2376 * skb_split - Split fragmented skb to two parts at length len.
2377 * @skb: the buffer to split
2378 * @skb1: the buffer to receive the second part
2379 * @len: new length for skb
2380 */
2381void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len)
2382{
2383 int pos = skb_headlen(skb);
2384
Amerigo Wang68534c62013-02-19 22:51:30 +00002385 skb_shinfo(skb1)->tx_flags = skb_shinfo(skb)->tx_flags & SKBTX_SHARED_FRAG;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002386 if (len < pos) /* Split line is inside header. */
2387 skb_split_inside_header(skb, skb1, len, pos);
2388 else /* Second chunk has no header, nothing to copy. */
2389 skb_split_no_header(skb, skb1, len, pos);
2390}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08002391EXPORT_SYMBOL(skb_split);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002392
Ilpo Järvinen9f782db2008-11-25 13:57:01 -08002393/* Shifting from/to a cloned skb is a no-go.
2394 *
2395 * Caller cannot keep skb_shinfo related pointers past calling here!
2396 */
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08002397static int skb_prepare_for_shift(struct sk_buff *skb)
2398{
Ilpo Järvinen0ace2852008-11-24 21:30:21 -08002399 return skb_cloned(skb) && pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08002400}
2401
2402/**
2403 * skb_shift - Shifts paged data partially from skb to another
2404 * @tgt: buffer into which tail data gets added
2405 * @skb: buffer from which the paged data comes from
2406 * @shiftlen: shift up to this many bytes
2407 *
2408 * Attempts to shift up to shiftlen worth of bytes, which may be less than
Feng King20e994a2011-11-21 01:47:11 +00002409 * the length of the skb, from skb to tgt. Returns number bytes shifted.
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08002410 * It's up to caller to free skb if everything was shifted.
2411 *
2412 * If @tgt runs out of frags, the whole operation is aborted.
2413 *
2414 * Skb cannot include anything else but paged data while tgt is allowed
2415 * to have non-paged data as well.
2416 *
2417 * TODO: full sized shift could be optimized but that would need
2418 * specialized skb free'er to handle frags without up-to-date nr_frags.
2419 */
2420int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen)
2421{
2422 int from, to, merge, todo;
2423 struct skb_frag_struct *fragfrom, *fragto;
2424
2425 BUG_ON(shiftlen > skb->len);
2426 BUG_ON(skb_headlen(skb)); /* Would corrupt stream */
2427
2428 todo = shiftlen;
2429 from = 0;
2430 to = skb_shinfo(tgt)->nr_frags;
2431 fragfrom = &skb_shinfo(skb)->frags[from];
2432
2433 /* Actual merge is delayed until the point when we know we can
2434 * commit all, so that we don't have to undo partial changes
2435 */
2436 if (!to ||
Ian Campbellea2ab692011-08-22 23:44:58 +00002437 !skb_can_coalesce(tgt, to, skb_frag_page(fragfrom),
2438 fragfrom->page_offset)) {
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08002439 merge = -1;
2440 } else {
2441 merge = to - 1;
2442
Eric Dumazet9e903e02011-10-18 21:00:24 +00002443 todo -= skb_frag_size(fragfrom);
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08002444 if (todo < 0) {
2445 if (skb_prepare_for_shift(skb) ||
2446 skb_prepare_for_shift(tgt))
2447 return 0;
2448
Ilpo Järvinen9f782db2008-11-25 13:57:01 -08002449 /* All previous frag pointers might be stale! */
2450 fragfrom = &skb_shinfo(skb)->frags[from];
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08002451 fragto = &skb_shinfo(tgt)->frags[merge];
2452
Eric Dumazet9e903e02011-10-18 21:00:24 +00002453 skb_frag_size_add(fragto, shiftlen);
2454 skb_frag_size_sub(fragfrom, shiftlen);
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08002455 fragfrom->page_offset += shiftlen;
2456
2457 goto onlymerged;
2458 }
2459
2460 from++;
2461 }
2462
2463 /* Skip full, not-fitting skb to avoid expensive operations */
2464 if ((shiftlen == skb->len) &&
2465 (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to))
2466 return 0;
2467
2468 if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt))
2469 return 0;
2470
2471 while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) {
2472 if (to == MAX_SKB_FRAGS)
2473 return 0;
2474
2475 fragfrom = &skb_shinfo(skb)->frags[from];
2476 fragto = &skb_shinfo(tgt)->frags[to];
2477
Eric Dumazet9e903e02011-10-18 21:00:24 +00002478 if (todo >= skb_frag_size(fragfrom)) {
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08002479 *fragto = *fragfrom;
Eric Dumazet9e903e02011-10-18 21:00:24 +00002480 todo -= skb_frag_size(fragfrom);
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08002481 from++;
2482 to++;
2483
2484 } else {
Ian Campbellea2ab692011-08-22 23:44:58 +00002485 __skb_frag_ref(fragfrom);
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08002486 fragto->page = fragfrom->page;
2487 fragto->page_offset = fragfrom->page_offset;
Eric Dumazet9e903e02011-10-18 21:00:24 +00002488 skb_frag_size_set(fragto, todo);
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08002489
2490 fragfrom->page_offset += todo;
Eric Dumazet9e903e02011-10-18 21:00:24 +00002491 skb_frag_size_sub(fragfrom, todo);
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08002492 todo = 0;
2493
2494 to++;
2495 break;
2496 }
2497 }
2498
2499 /* Ready to "commit" this state change to tgt */
2500 skb_shinfo(tgt)->nr_frags = to;
2501
2502 if (merge >= 0) {
2503 fragfrom = &skb_shinfo(skb)->frags[0];
2504 fragto = &skb_shinfo(tgt)->frags[merge];
2505
Eric Dumazet9e903e02011-10-18 21:00:24 +00002506 skb_frag_size_add(fragto, skb_frag_size(fragfrom));
Ian Campbellea2ab692011-08-22 23:44:58 +00002507 __skb_frag_unref(fragfrom);
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08002508 }
2509
2510 /* Reposition in the original skb */
2511 to = 0;
2512 while (from < skb_shinfo(skb)->nr_frags)
2513 skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++];
2514 skb_shinfo(skb)->nr_frags = to;
2515
2516 BUG_ON(todo > 0 && !skb_shinfo(skb)->nr_frags);
2517
2518onlymerged:
2519 /* Most likely the tgt won't ever need its checksum anymore, skb on
2520 * the other hand might need it if it needs to be resent
2521 */
2522 tgt->ip_summed = CHECKSUM_PARTIAL;
2523 skb->ip_summed = CHECKSUM_PARTIAL;
2524
2525 /* Yak, is it really working this way? Some helper please? */
2526 skb->len -= shiftlen;
2527 skb->data_len -= shiftlen;
2528 skb->truesize -= shiftlen;
2529 tgt->len += shiftlen;
2530 tgt->data_len += shiftlen;
2531 tgt->truesize += shiftlen;
2532
2533 return shiftlen;
2534}
2535
Thomas Graf677e90e2005-06-23 20:59:51 -07002536/**
2537 * skb_prepare_seq_read - Prepare a sequential read of skb data
2538 * @skb: the buffer to read
2539 * @from: lower offset of data to be read
2540 * @to: upper offset of data to be read
2541 * @st: state variable
2542 *
2543 * Initializes the specified state variable. Must be called before
2544 * invoking skb_seq_read() for the first time.
2545 */
2546void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
2547 unsigned int to, struct skb_seq_state *st)
2548{
2549 st->lower_offset = from;
2550 st->upper_offset = to;
2551 st->root_skb = st->cur_skb = skb;
2552 st->frag_idx = st->stepped_offset = 0;
2553 st->frag_data = NULL;
2554}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08002555EXPORT_SYMBOL(skb_prepare_seq_read);
Thomas Graf677e90e2005-06-23 20:59:51 -07002556
2557/**
2558 * skb_seq_read - Sequentially read skb data
2559 * @consumed: number of bytes consumed by the caller so far
2560 * @data: destination pointer for data to be returned
2561 * @st: state variable
2562 *
Mathias Krausebc323832013-11-07 14:18:26 +01002563 * Reads a block of skb data at @consumed relative to the
Thomas Graf677e90e2005-06-23 20:59:51 -07002564 * lower offset specified to skb_prepare_seq_read(). Assigns
Mathias Krausebc323832013-11-07 14:18:26 +01002565 * the head of the data block to @data and returns the length
Thomas Graf677e90e2005-06-23 20:59:51 -07002566 * of the block or 0 if the end of the skb data or the upper
2567 * offset has been reached.
2568 *
2569 * The caller is not required to consume all of the data
Mathias Krausebc323832013-11-07 14:18:26 +01002570 * returned, i.e. @consumed is typically set to the number
Thomas Graf677e90e2005-06-23 20:59:51 -07002571 * of bytes already consumed and the next call to
2572 * skb_seq_read() will return the remaining part of the block.
2573 *
Lucas De Marchi25985ed2011-03-30 22:57:33 -03002574 * Note 1: The size of each block of data returned can be arbitrary,
Thomas Graf677e90e2005-06-23 20:59:51 -07002575 * this limitation is the cost for zerocopy seqeuental
2576 * reads of potentially non linear data.
2577 *
Randy Dunlapbc2cda12008-02-13 15:03:25 -08002578 * Note 2: Fragment lists within fragments are not implemented
Thomas Graf677e90e2005-06-23 20:59:51 -07002579 * at the moment, state->root_skb could be replaced with
2580 * a stack for this purpose.
2581 */
2582unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
2583 struct skb_seq_state *st)
2584{
2585 unsigned int block_limit, abs_offset = consumed + st->lower_offset;
2586 skb_frag_t *frag;
2587
Wedson Almeida Filhoaeb193e2013-06-23 23:33:48 -07002588 if (unlikely(abs_offset >= st->upper_offset)) {
2589 if (st->frag_data) {
2590 kunmap_atomic(st->frag_data);
2591 st->frag_data = NULL;
2592 }
Thomas Graf677e90e2005-06-23 20:59:51 -07002593 return 0;
Wedson Almeida Filhoaeb193e2013-06-23 23:33:48 -07002594 }
Thomas Graf677e90e2005-06-23 20:59:51 -07002595
2596next_skb:
Herbert Xu95e3b242009-01-29 16:07:52 -08002597 block_limit = skb_headlen(st->cur_skb) + st->stepped_offset;
Thomas Graf677e90e2005-06-23 20:59:51 -07002598
Thomas Chenault995b3372009-05-18 21:43:27 -07002599 if (abs_offset < block_limit && !st->frag_data) {
Herbert Xu95e3b242009-01-29 16:07:52 -08002600 *data = st->cur_skb->data + (abs_offset - st->stepped_offset);
Thomas Graf677e90e2005-06-23 20:59:51 -07002601 return block_limit - abs_offset;
2602 }
2603
2604 if (st->frag_idx == 0 && !st->frag_data)
2605 st->stepped_offset += skb_headlen(st->cur_skb);
2606
2607 while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) {
2608 frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx];
Eric Dumazet9e903e02011-10-18 21:00:24 +00002609 block_limit = skb_frag_size(frag) + st->stepped_offset;
Thomas Graf677e90e2005-06-23 20:59:51 -07002610
2611 if (abs_offset < block_limit) {
2612 if (!st->frag_data)
Eric Dumazet51c56b02012-04-05 11:35:15 +02002613 st->frag_data = kmap_atomic(skb_frag_page(frag));
Thomas Graf677e90e2005-06-23 20:59:51 -07002614
2615 *data = (u8 *) st->frag_data + frag->page_offset +
2616 (abs_offset - st->stepped_offset);
2617
2618 return block_limit - abs_offset;
2619 }
2620
2621 if (st->frag_data) {
Eric Dumazet51c56b02012-04-05 11:35:15 +02002622 kunmap_atomic(st->frag_data);
Thomas Graf677e90e2005-06-23 20:59:51 -07002623 st->frag_data = NULL;
2624 }
2625
2626 st->frag_idx++;
Eric Dumazet9e903e02011-10-18 21:00:24 +00002627 st->stepped_offset += skb_frag_size(frag);
Thomas Graf677e90e2005-06-23 20:59:51 -07002628 }
2629
Olaf Kirch5b5a60d2007-06-23 23:11:52 -07002630 if (st->frag_data) {
Eric Dumazet51c56b02012-04-05 11:35:15 +02002631 kunmap_atomic(st->frag_data);
Olaf Kirch5b5a60d2007-06-23 23:11:52 -07002632 st->frag_data = NULL;
2633 }
2634
David S. Miller21dc3302010-08-23 00:13:46 -07002635 if (st->root_skb == st->cur_skb && skb_has_frag_list(st->root_skb)) {
Shyam Iyer71b33462009-01-29 16:12:42 -08002636 st->cur_skb = skb_shinfo(st->root_skb)->frag_list;
Thomas Graf677e90e2005-06-23 20:59:51 -07002637 st->frag_idx = 0;
2638 goto next_skb;
Shyam Iyer71b33462009-01-29 16:12:42 -08002639 } else if (st->cur_skb->next) {
2640 st->cur_skb = st->cur_skb->next;
Herbert Xu95e3b242009-01-29 16:07:52 -08002641 st->frag_idx = 0;
Thomas Graf677e90e2005-06-23 20:59:51 -07002642 goto next_skb;
2643 }
2644
2645 return 0;
2646}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08002647EXPORT_SYMBOL(skb_seq_read);
Thomas Graf677e90e2005-06-23 20:59:51 -07002648
2649/**
2650 * skb_abort_seq_read - Abort a sequential read of skb data
2651 * @st: state variable
2652 *
2653 * Must be called if skb_seq_read() was not called until it
2654 * returned 0.
2655 */
2656void skb_abort_seq_read(struct skb_seq_state *st)
2657{
2658 if (st->frag_data)
Eric Dumazet51c56b02012-04-05 11:35:15 +02002659 kunmap_atomic(st->frag_data);
Thomas Graf677e90e2005-06-23 20:59:51 -07002660}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08002661EXPORT_SYMBOL(skb_abort_seq_read);
Thomas Graf677e90e2005-06-23 20:59:51 -07002662
Thomas Graf3fc7e8a2005-06-23 21:00:17 -07002663#define TS_SKB_CB(state) ((struct skb_seq_state *) &((state)->cb))
2664
2665static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text,
2666 struct ts_config *conf,
2667 struct ts_state *state)
2668{
2669 return skb_seq_read(offset, text, TS_SKB_CB(state));
2670}
2671
2672static void skb_ts_finish(struct ts_config *conf, struct ts_state *state)
2673{
2674 skb_abort_seq_read(TS_SKB_CB(state));
2675}
2676
2677/**
2678 * skb_find_text - Find a text pattern in skb data
2679 * @skb: the buffer to look in
2680 * @from: search offset
2681 * @to: search limit
2682 * @config: textsearch configuration
2683 * @state: uninitialized textsearch state variable
2684 *
2685 * Finds a pattern in the skb data according to the specified
2686 * textsearch configuration. Use textsearch_next() to retrieve
2687 * subsequent occurrences of the pattern. Returns the offset
2688 * to the first occurrence or UINT_MAX if no match was found.
2689 */
2690unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
2691 unsigned int to, struct ts_config *config,
2692 struct ts_state *state)
2693{
Phil Oesterf72b9482006-06-26 00:00:57 -07002694 unsigned int ret;
2695
Thomas Graf3fc7e8a2005-06-23 21:00:17 -07002696 config->get_next_block = skb_ts_get_next_block;
2697 config->finish = skb_ts_finish;
2698
2699 skb_prepare_seq_read(skb, from, to, TS_SKB_CB(state));
2700
Phil Oesterf72b9482006-06-26 00:00:57 -07002701 ret = textsearch_find(config, state);
2702 return (ret <= to - from ? ret : UINT_MAX);
Thomas Graf3fc7e8a2005-06-23 21:00:17 -07002703}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08002704EXPORT_SYMBOL(skb_find_text);
Thomas Graf3fc7e8a2005-06-23 21:00:17 -07002705
Ananda Rajue89e9cf2005-10-18 15:46:41 -07002706/**
Ben Hutchings2c530402012-07-10 10:55:09 +00002707 * skb_append_datato_frags - append the user data to a skb
Ananda Rajue89e9cf2005-10-18 15:46:41 -07002708 * @sk: sock structure
2709 * @skb: skb structure to be appened with user data.
2710 * @getfrag: call back function to be used for getting the user data
2711 * @from: pointer to user message iov
2712 * @length: length of the iov message
2713 *
2714 * Description: This procedure append the user data in the fragment part
2715 * of the skb if any page alloc fails user this procedure returns -ENOMEM
2716 */
2717int skb_append_datato_frags(struct sock *sk, struct sk_buff *skb,
Martin Waitzdab96302005-12-05 13:40:12 -08002718 int (*getfrag)(void *from, char *to, int offset,
Ananda Rajue89e9cf2005-10-18 15:46:41 -07002719 int len, int odd, struct sk_buff *skb),
2720 void *from, int length)
2721{
Eric Dumazetb2111722012-12-28 06:06:37 +00002722 int frg_cnt = skb_shinfo(skb)->nr_frags;
2723 int copy;
Ananda Rajue89e9cf2005-10-18 15:46:41 -07002724 int offset = 0;
2725 int ret;
Eric Dumazetb2111722012-12-28 06:06:37 +00002726 struct page_frag *pfrag = &current->task_frag;
Ananda Rajue89e9cf2005-10-18 15:46:41 -07002727
2728 do {
2729 /* Return error if we don't have space for new frag */
Ananda Rajue89e9cf2005-10-18 15:46:41 -07002730 if (frg_cnt >= MAX_SKB_FRAGS)
Eric Dumazetb2111722012-12-28 06:06:37 +00002731 return -EMSGSIZE;
Ananda Rajue89e9cf2005-10-18 15:46:41 -07002732
Eric Dumazetb2111722012-12-28 06:06:37 +00002733 if (!sk_page_frag_refill(sk, pfrag))
Ananda Rajue89e9cf2005-10-18 15:46:41 -07002734 return -ENOMEM;
2735
Ananda Rajue89e9cf2005-10-18 15:46:41 -07002736 /* copy the user data to page */
Eric Dumazetb2111722012-12-28 06:06:37 +00002737 copy = min_t(int, length, pfrag->size - pfrag->offset);
Ananda Rajue89e9cf2005-10-18 15:46:41 -07002738
Eric Dumazetb2111722012-12-28 06:06:37 +00002739 ret = getfrag(from, page_address(pfrag->page) + pfrag->offset,
2740 offset, copy, 0, skb);
Ananda Rajue89e9cf2005-10-18 15:46:41 -07002741 if (ret < 0)
2742 return -EFAULT;
2743
2744 /* copy was successful so update the size parameters */
Eric Dumazetb2111722012-12-28 06:06:37 +00002745 skb_fill_page_desc(skb, frg_cnt, pfrag->page, pfrag->offset,
2746 copy);
2747 frg_cnt++;
2748 pfrag->offset += copy;
2749 get_page(pfrag->page);
2750
2751 skb->truesize += copy;
2752 atomic_add(copy, &sk->sk_wmem_alloc);
Ananda Rajue89e9cf2005-10-18 15:46:41 -07002753 skb->len += copy;
2754 skb->data_len += copy;
2755 offset += copy;
2756 length -= copy;
2757
2758 } while (length > 0);
2759
2760 return 0;
2761}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08002762EXPORT_SYMBOL(skb_append_datato_frags);
Ananda Rajue89e9cf2005-10-18 15:46:41 -07002763
Herbert Xucbb042f2006-03-20 22:43:56 -08002764/**
2765 * skb_pull_rcsum - pull skb and update receive checksum
2766 * @skb: buffer to update
Herbert Xucbb042f2006-03-20 22:43:56 -08002767 * @len: length of data pulled
2768 *
2769 * This function performs an skb_pull on the packet and updates
Urs Thuermannfee54fa2008-02-12 22:03:25 -08002770 * the CHECKSUM_COMPLETE checksum. It should be used on
Patrick McHardy84fa7932006-08-29 16:44:56 -07002771 * receive path processing instead of skb_pull unless you know
2772 * that the checksum difference is zero (e.g., a valid IP header)
2773 * or you are setting ip_summed to CHECKSUM_NONE.
Herbert Xucbb042f2006-03-20 22:43:56 -08002774 */
2775unsigned char *skb_pull_rcsum(struct sk_buff *skb, unsigned int len)
2776{
2777 BUG_ON(len > skb->len);
2778 skb->len -= len;
2779 BUG_ON(skb->len < skb->data_len);
2780 skb_postpull_rcsum(skb, skb->data, len);
2781 return skb->data += len;
2782}
Arnaldo Carvalho de Melof94691a2006-03-20 22:47:55 -08002783EXPORT_SYMBOL_GPL(skb_pull_rcsum);
2784
Herbert Xuf4c50d92006-06-22 03:02:40 -07002785/**
2786 * skb_segment - Perform protocol segmentation on skb.
2787 * @skb: buffer to segment
Herbert Xu576a30e2006-06-27 13:22:38 -07002788 * @features: features for the output path (see dev->features)
Herbert Xuf4c50d92006-06-22 03:02:40 -07002789 *
2790 * This function performs segmentation on the given skb. It returns
Ben Hutchings4c821d72008-04-13 21:52:48 -07002791 * a pointer to the first in a list of new skbs for the segments.
2792 * In case of error it returns ERR_PTR(err).
Herbert Xuf4c50d92006-06-22 03:02:40 -07002793 */
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002794struct sk_buff *skb_segment(struct sk_buff *skb, netdev_features_t features)
Herbert Xuf4c50d92006-06-22 03:02:40 -07002795{
2796 struct sk_buff *segs = NULL;
2797 struct sk_buff *tail = NULL;
Herbert Xu89319d382008-12-15 23:26:06 -08002798 struct sk_buff *fskb = skb_shinfo(skb)->frag_list;
Herbert Xu9d8506c2013-11-21 11:10:04 -08002799 skb_frag_t *skb_frag = skb_shinfo(skb)->frags;
Herbert Xuf4c50d92006-06-22 03:02:40 -07002800 unsigned int mss = skb_shinfo(skb)->gso_size;
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -07002801 unsigned int doffset = skb->data - skb_mac_header(skb);
Herbert Xuf4c50d92006-06-22 03:02:40 -07002802 unsigned int offset = doffset;
Pravin B Shelar68c33162013-02-14 14:02:41 +00002803 unsigned int tnl_hlen = skb_tnl_header_len(skb);
Herbert Xuf4c50d92006-06-22 03:02:40 -07002804 unsigned int headroom;
2805 unsigned int len;
Pravin B Shelarec5f0612013-03-07 09:28:01 +00002806 __be16 proto;
2807 bool csum;
Michał Mirosław04ed3e72011-01-24 15:32:47 -08002808 int sg = !!(features & NETIF_F_SG);
Herbert Xuf4c50d92006-06-22 03:02:40 -07002809 int nfrags = skb_shinfo(skb)->nr_frags;
2810 int err = -ENOMEM;
2811 int i = 0;
2812 int pos;
2813
Pravin B Shelarec5f0612013-03-07 09:28:01 +00002814 proto = skb_network_protocol(skb);
2815 if (unlikely(!proto))
2816 return ERR_PTR(-EINVAL);
2817
2818 csum = !!can_checksum_protocol(features, proto);
Herbert Xuf4c50d92006-06-22 03:02:40 -07002819 __skb_push(skb, doffset);
2820 headroom = skb_headroom(skb);
2821 pos = skb_headlen(skb);
2822
2823 do {
2824 struct sk_buff *nskb;
2825 skb_frag_t *frag;
Herbert Xuc8884ed2006-10-29 15:59:41 -08002826 int hsize;
Herbert Xuf4c50d92006-06-22 03:02:40 -07002827 int size;
2828
2829 len = skb->len - offset;
2830 if (len > mss)
2831 len = mss;
2832
2833 hsize = skb_headlen(skb) - offset;
2834 if (hsize < 0)
2835 hsize = 0;
Herbert Xuc8884ed2006-10-29 15:59:41 -08002836 if (hsize > len || !sg)
2837 hsize = len;
Herbert Xuf4c50d92006-06-22 03:02:40 -07002838
Herbert Xu9d8506c2013-11-21 11:10:04 -08002839 if (!hsize && i >= nfrags && skb_headlen(fskb) &&
2840 (skb_headlen(fskb) == len || sg)) {
2841 BUG_ON(skb_headlen(fskb) > len);
Herbert Xu89319d382008-12-15 23:26:06 -08002842
Herbert Xu9d8506c2013-11-21 11:10:04 -08002843 i = 0;
2844 nfrags = skb_shinfo(fskb)->nr_frags;
2845 skb_frag = skb_shinfo(fskb)->frags;
2846 pos += skb_headlen(fskb);
2847
2848 while (pos < offset + len) {
2849 BUG_ON(i >= nfrags);
2850
2851 size = skb_frag_size(skb_frag);
2852 if (pos + size > offset + len)
2853 break;
2854
2855 i++;
2856 pos += size;
2857 skb_frag++;
2858 }
2859
Herbert Xu89319d382008-12-15 23:26:06 -08002860 nskb = skb_clone(fskb, GFP_ATOMIC);
2861 fskb = fskb->next;
2862
2863 if (unlikely(!nskb))
2864 goto err;
2865
Herbert Xu9d8506c2013-11-21 11:10:04 -08002866 if (unlikely(pskb_trim(nskb, len))) {
2867 kfree_skb(nskb);
2868 goto err;
2869 }
2870
Alexander Duyckec47ea82012-05-04 14:26:56 +00002871 hsize = skb_end_offset(nskb);
Herbert Xu89319d382008-12-15 23:26:06 -08002872 if (skb_cow_head(nskb, doffset + headroom)) {
2873 kfree_skb(nskb);
2874 goto err;
2875 }
2876
Alexander Duyckec47ea82012-05-04 14:26:56 +00002877 nskb->truesize += skb_end_offset(nskb) - hsize;
Herbert Xu89319d382008-12-15 23:26:06 -08002878 skb_release_head_state(nskb);
2879 __skb_push(nskb, doffset);
2880 } else {
Mel Gormanc93bdd02012-07-31 16:44:19 -07002881 nskb = __alloc_skb(hsize + doffset + headroom,
2882 GFP_ATOMIC, skb_alloc_rx_flag(skb),
2883 NUMA_NO_NODE);
Herbert Xu89319d382008-12-15 23:26:06 -08002884
2885 if (unlikely(!nskb))
2886 goto err;
2887
2888 skb_reserve(nskb, headroom);
2889 __skb_put(nskb, doffset);
2890 }
Herbert Xuf4c50d92006-06-22 03:02:40 -07002891
2892 if (segs)
2893 tail->next = nskb;
2894 else
2895 segs = nskb;
2896 tail = nskb;
2897
Herbert Xu6f85a122008-08-15 14:55:02 -07002898 __copy_skb_header(nskb, skb);
Herbert Xuf4c50d92006-06-22 03:02:40 -07002899 nskb->mac_len = skb->mac_len;
2900
Eric Dumazet030737b2013-10-19 11:42:54 -07002901 skb_headers_offset_update(nskb, skb_headroom(nskb) - headroom);
Pravin B Shelar68c33162013-02-14 14:02:41 +00002902
2903 skb_copy_from_linear_data_offset(skb, -tnl_hlen,
2904 nskb->data - tnl_hlen,
2905 doffset + tnl_hlen);
Herbert Xu89319d382008-12-15 23:26:06 -08002906
Herbert Xu9d8506c2013-11-21 11:10:04 -08002907 if (nskb->len == len + doffset)
Simon Horman1cdbcb72013-05-19 15:46:49 +00002908 goto perform_csum_check;
Herbert Xu89319d382008-12-15 23:26:06 -08002909
Herbert Xuf4c50d92006-06-22 03:02:40 -07002910 if (!sg) {
Herbert Xu6f85a122008-08-15 14:55:02 -07002911 nskb->ip_summed = CHECKSUM_NONE;
Herbert Xuf4c50d92006-06-22 03:02:40 -07002912 nskb->csum = skb_copy_and_csum_bits(skb, offset,
2913 skb_put(nskb, len),
2914 len, 0);
2915 continue;
2916 }
2917
2918 frag = skb_shinfo(nskb)->frags;
Herbert Xuf4c50d92006-06-22 03:02:40 -07002919
Arnaldo Carvalho de Melod626f622007-03-27 18:55:52 -03002920 skb_copy_from_linear_data_offset(skb, offset,
2921 skb_put(nskb, hsize), hsize);
Herbert Xuf4c50d92006-06-22 03:02:40 -07002922
Pravin B Shelarc9af6db2013-02-11 09:27:41 +00002923 skb_shinfo(nskb)->tx_flags = skb_shinfo(skb)->tx_flags & SKBTX_SHARED_FRAG;
Eric Dumazetcef401d2013-01-25 20:34:37 +00002924
Herbert Xu9d8506c2013-11-21 11:10:04 -08002925 while (pos < offset + len) {
2926 if (i >= nfrags) {
2927 BUG_ON(skb_headlen(fskb));
2928
2929 i = 0;
2930 nfrags = skb_shinfo(fskb)->nr_frags;
2931 skb_frag = skb_shinfo(fskb)->frags;
2932
2933 BUG_ON(!nfrags);
2934
2935 fskb = fskb->next;
2936 }
2937
2938 if (unlikely(skb_shinfo(nskb)->nr_frags >=
2939 MAX_SKB_FRAGS)) {
2940 net_warn_ratelimited(
2941 "skb_segment: too many frags: %u %u\n",
2942 pos, mss);
2943 goto err;
2944 }
2945
2946 *frag = *skb_frag;
Ian Campbellea2ab692011-08-22 23:44:58 +00002947 __skb_frag_ref(frag);
Eric Dumazet9e903e02011-10-18 21:00:24 +00002948 size = skb_frag_size(frag);
Herbert Xuf4c50d92006-06-22 03:02:40 -07002949
2950 if (pos < offset) {
2951 frag->page_offset += offset - pos;
Eric Dumazet9e903e02011-10-18 21:00:24 +00002952 skb_frag_size_sub(frag, offset - pos);
Herbert Xuf4c50d92006-06-22 03:02:40 -07002953 }
2954
Herbert Xu89319d382008-12-15 23:26:06 -08002955 skb_shinfo(nskb)->nr_frags++;
Herbert Xuf4c50d92006-06-22 03:02:40 -07002956
2957 if (pos + size <= offset + len) {
2958 i++;
Herbert Xu9d8506c2013-11-21 11:10:04 -08002959 skb_frag++;
Herbert Xuf4c50d92006-06-22 03:02:40 -07002960 pos += size;
2961 } else {
Eric Dumazet9e903e02011-10-18 21:00:24 +00002962 skb_frag_size_sub(frag, pos + size - (offset + len));
Herbert Xu89319d382008-12-15 23:26:06 -08002963 goto skip_fraglist;
Herbert Xuf4c50d92006-06-22 03:02:40 -07002964 }
2965
2966 frag++;
2967 }
2968
Herbert Xu89319d382008-12-15 23:26:06 -08002969skip_fraglist:
Herbert Xuf4c50d92006-06-22 03:02:40 -07002970 nskb->data_len = len - hsize;
2971 nskb->len += nskb->data_len;
2972 nskb->truesize += nskb->data_len;
Pravin B Shelarec5f0612013-03-07 09:28:01 +00002973
Simon Horman1cdbcb72013-05-19 15:46:49 +00002974perform_csum_check:
Pravin B Shelarec5f0612013-03-07 09:28:01 +00002975 if (!csum) {
2976 nskb->csum = skb_checksum(nskb, doffset,
2977 nskb->len - doffset, 0);
2978 nskb->ip_summed = CHECKSUM_NONE;
2979 }
Herbert Xuf4c50d92006-06-22 03:02:40 -07002980 } while ((offset += len) < skb->len);
2981
2982 return segs;
2983
2984err:
2985 while ((skb = segs)) {
2986 segs = skb->next;
Patrick McHardyb08d5842007-02-27 09:57:37 -08002987 kfree_skb(skb);
Herbert Xuf4c50d92006-06-22 03:02:40 -07002988 }
2989 return ERR_PTR(err);
2990}
Herbert Xuf4c50d92006-06-22 03:02:40 -07002991EXPORT_SYMBOL_GPL(skb_segment);
2992
Herbert Xu71d93b32008-12-15 23:42:33 -08002993int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb)
2994{
Eric Dumazet8a291112013-10-08 09:02:23 -07002995 struct skb_shared_info *pinfo, *skbinfo = skb_shinfo(skb);
Herbert Xu67147ba2009-05-26 18:50:22 +00002996 unsigned int offset = skb_gro_offset(skb);
2997 unsigned int headlen = skb_headlen(skb);
Eric Dumazet8a291112013-10-08 09:02:23 -07002998 struct sk_buff *nskb, *lp, *p = *head;
2999 unsigned int len = skb_gro_len(skb);
Eric Dumazet715dc1f2012-05-02 23:33:21 +00003000 unsigned int delta_truesize;
Eric Dumazet8a291112013-10-08 09:02:23 -07003001 unsigned int headroom;
Herbert Xu71d93b32008-12-15 23:42:33 -08003002
Eric Dumazet8a291112013-10-08 09:02:23 -07003003 if (unlikely(p->len + len >= 65536))
Herbert Xu71d93b32008-12-15 23:42:33 -08003004 return -E2BIG;
3005
Eric Dumazet8a291112013-10-08 09:02:23 -07003006 lp = NAPI_GRO_CB(p)->last ?: p;
3007 pinfo = skb_shinfo(lp);
3008
3009 if (headlen <= offset) {
Herbert Xu42da6992009-05-26 18:50:19 +00003010 skb_frag_t *frag;
Herbert Xu66e92fc2009-05-26 18:50:32 +00003011 skb_frag_t *frag2;
Herbert Xu9aaa1562009-05-26 18:50:33 +00003012 int i = skbinfo->nr_frags;
3013 int nr_frags = pinfo->nr_frags + i;
Herbert Xu42da6992009-05-26 18:50:19 +00003014
Herbert Xu66e92fc2009-05-26 18:50:32 +00003015 if (nr_frags > MAX_SKB_FRAGS)
Eric Dumazet8a291112013-10-08 09:02:23 -07003016 goto merge;
Herbert Xu81705ad2009-01-29 14:19:51 +00003017
Eric Dumazet8a291112013-10-08 09:02:23 -07003018 offset -= headlen;
Herbert Xu9aaa1562009-05-26 18:50:33 +00003019 pinfo->nr_frags = nr_frags;
3020 skbinfo->nr_frags = 0;
Herbert Xuf5572062009-01-14 20:40:03 -08003021
Herbert Xu9aaa1562009-05-26 18:50:33 +00003022 frag = pinfo->frags + nr_frags;
3023 frag2 = skbinfo->frags + i;
Herbert Xu66e92fc2009-05-26 18:50:32 +00003024 do {
3025 *--frag = *--frag2;
3026 } while (--i);
3027
3028 frag->page_offset += offset;
Eric Dumazet9e903e02011-10-18 21:00:24 +00003029 skb_frag_size_sub(frag, offset);
Herbert Xu66e92fc2009-05-26 18:50:32 +00003030
Eric Dumazet715dc1f2012-05-02 23:33:21 +00003031 /* all fragments truesize : remove (head size + sk_buff) */
Alexander Duyckec47ea82012-05-04 14:26:56 +00003032 delta_truesize = skb->truesize -
3033 SKB_TRUESIZE(skb_end_offset(skb));
Eric Dumazet715dc1f2012-05-02 23:33:21 +00003034
Herbert Xuf5572062009-01-14 20:40:03 -08003035 skb->truesize -= skb->data_len;
3036 skb->len -= skb->data_len;
3037 skb->data_len = 0;
3038
Eric Dumazet715dc1f2012-05-02 23:33:21 +00003039 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE;
Herbert Xu5d38a072009-01-04 16:13:40 -08003040 goto done;
Eric Dumazetd7e88832012-04-30 08:10:34 +00003041 } else if (skb->head_frag) {
3042 int nr_frags = pinfo->nr_frags;
3043 skb_frag_t *frag = pinfo->frags + nr_frags;
3044 struct page *page = virt_to_head_page(skb->head);
3045 unsigned int first_size = headlen - offset;
3046 unsigned int first_offset;
3047
3048 if (nr_frags + 1 + skbinfo->nr_frags > MAX_SKB_FRAGS)
Eric Dumazet8a291112013-10-08 09:02:23 -07003049 goto merge;
Eric Dumazetd7e88832012-04-30 08:10:34 +00003050
3051 first_offset = skb->data -
3052 (unsigned char *)page_address(page) +
3053 offset;
3054
3055 pinfo->nr_frags = nr_frags + 1 + skbinfo->nr_frags;
3056
3057 frag->page.p = page;
3058 frag->page_offset = first_offset;
3059 skb_frag_size_set(frag, first_size);
3060
3061 memcpy(frag + 1, skbinfo->frags, sizeof(*frag) * skbinfo->nr_frags);
3062 /* We dont need to clear skbinfo->nr_frags here */
3063
Eric Dumazet715dc1f2012-05-02 23:33:21 +00003064 delta_truesize = skb->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff));
Eric Dumazetd7e88832012-04-30 08:10:34 +00003065 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE_STOLEN_HEAD;
3066 goto done;
Eric Dumazet8a291112013-10-08 09:02:23 -07003067 }
3068 if (pinfo->frag_list)
3069 goto merge;
3070 if (skb_gro_len(p) != pinfo->gso_size)
Herbert Xu69c0cab2009-11-17 05:18:18 -08003071 return -E2BIG;
Herbert Xu71d93b32008-12-15 23:42:33 -08003072
3073 headroom = skb_headroom(p);
Eric Dumazet3d3be432010-09-01 00:50:51 +00003074 nskb = alloc_skb(headroom + skb_gro_offset(p), GFP_ATOMIC);
Herbert Xu71d93b32008-12-15 23:42:33 -08003075 if (unlikely(!nskb))
3076 return -ENOMEM;
3077
3078 __copy_skb_header(nskb, p);
3079 nskb->mac_len = p->mac_len;
3080
3081 skb_reserve(nskb, headroom);
Herbert Xu86911732009-01-29 14:19:50 +00003082 __skb_put(nskb, skb_gro_offset(p));
Herbert Xu71d93b32008-12-15 23:42:33 -08003083
Herbert Xu86911732009-01-29 14:19:50 +00003084 skb_set_mac_header(nskb, skb_mac_header(p) - p->data);
Herbert Xu71d93b32008-12-15 23:42:33 -08003085 skb_set_network_header(nskb, skb_network_offset(p));
3086 skb_set_transport_header(nskb, skb_transport_offset(p));
3087
Herbert Xu86911732009-01-29 14:19:50 +00003088 __skb_pull(p, skb_gro_offset(p));
3089 memcpy(skb_mac_header(nskb), skb_mac_header(p),
3090 p->data - skb_mac_header(p));
Herbert Xu71d93b32008-12-15 23:42:33 -08003091
Herbert Xu71d93b32008-12-15 23:42:33 -08003092 skb_shinfo(nskb)->frag_list = p;
Herbert Xu9aaa1562009-05-26 18:50:33 +00003093 skb_shinfo(nskb)->gso_size = pinfo->gso_size;
Herbert Xu622e0ca2010-05-20 23:07:56 -07003094 pinfo->gso_size = 0;
Herbert Xu71d93b32008-12-15 23:42:33 -08003095 skb_header_release(p);
Eric Dumazetc3c7c252012-12-06 13:54:59 +00003096 NAPI_GRO_CB(nskb)->last = p;
Herbert Xu71d93b32008-12-15 23:42:33 -08003097
3098 nskb->data_len += p->len;
Eric Dumazetde8261c2012-02-13 04:09:20 +00003099 nskb->truesize += p->truesize;
Herbert Xu71d93b32008-12-15 23:42:33 -08003100 nskb->len += p->len;
3101
3102 *head = nskb;
3103 nskb->next = p->next;
3104 p->next = NULL;
3105
3106 p = nskb;
3107
3108merge:
Eric Dumazet715dc1f2012-05-02 23:33:21 +00003109 delta_truesize = skb->truesize;
Herbert Xu67147ba2009-05-26 18:50:22 +00003110 if (offset > headlen) {
Michal Schmidtd1dc7ab2011-01-24 12:08:48 +00003111 unsigned int eat = offset - headlen;
3112
3113 skbinfo->frags[0].page_offset += eat;
Eric Dumazet9e903e02011-10-18 21:00:24 +00003114 skb_frag_size_sub(&skbinfo->frags[0], eat);
Michal Schmidtd1dc7ab2011-01-24 12:08:48 +00003115 skb->data_len -= eat;
3116 skb->len -= eat;
Herbert Xu67147ba2009-05-26 18:50:22 +00003117 offset = headlen;
Herbert Xu56035022009-02-05 21:26:52 -08003118 }
3119
Herbert Xu67147ba2009-05-26 18:50:22 +00003120 __skb_pull(skb, offset);
Herbert Xu56035022009-02-05 21:26:52 -08003121
Eric Dumazet8a291112013-10-08 09:02:23 -07003122 if (!NAPI_GRO_CB(p)->last)
3123 skb_shinfo(p)->frag_list = skb;
3124 else
3125 NAPI_GRO_CB(p)->last->next = skb;
Eric Dumazetc3c7c252012-12-06 13:54:59 +00003126 NAPI_GRO_CB(p)->last = skb;
Herbert Xu71d93b32008-12-15 23:42:33 -08003127 skb_header_release(skb);
Eric Dumazet8a291112013-10-08 09:02:23 -07003128 lp = p;
Herbert Xu71d93b32008-12-15 23:42:33 -08003129
Herbert Xu5d38a072009-01-04 16:13:40 -08003130done:
3131 NAPI_GRO_CB(p)->count++;
Herbert Xu37fe4732009-01-17 19:48:13 +00003132 p->data_len += len;
Eric Dumazet715dc1f2012-05-02 23:33:21 +00003133 p->truesize += delta_truesize;
Herbert Xu37fe4732009-01-17 19:48:13 +00003134 p->len += len;
Eric Dumazet8a291112013-10-08 09:02:23 -07003135 if (lp != p) {
3136 lp->data_len += len;
3137 lp->truesize += delta_truesize;
3138 lp->len += len;
3139 }
Herbert Xu71d93b32008-12-15 23:42:33 -08003140 NAPI_GRO_CB(skb)->same_flow = 1;
3141 return 0;
3142}
3143EXPORT_SYMBOL_GPL(skb_gro_receive);
3144
Linus Torvalds1da177e2005-04-16 15:20:36 -07003145void __init skb_init(void)
3146{
3147 skbuff_head_cache = kmem_cache_create("skbuff_head_cache",
3148 sizeof(struct sk_buff),
3149 0,
Alexey Dobriyane5d679f332006-08-26 19:25:52 -07003150 SLAB_HWCACHE_ALIGN|SLAB_PANIC,
Paul Mundt20c2df82007-07-20 10:11:58 +09003151 NULL);
David S. Millerd179cd12005-08-17 14:57:30 -07003152 skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache",
3153 (2*sizeof(struct sk_buff)) +
3154 sizeof(atomic_t),
3155 0,
Alexey Dobriyane5d679f332006-08-26 19:25:52 -07003156 SLAB_HWCACHE_ALIGN|SLAB_PANIC,
Paul Mundt20c2df82007-07-20 10:11:58 +09003157 NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003158}
3159
David Howells716ea3a2007-04-02 20:19:53 -07003160/**
3161 * skb_to_sgvec - Fill a scatter-gather list from a socket buffer
3162 * @skb: Socket buffer containing the buffers to be mapped
3163 * @sg: The scatter-gather list to map into
3164 * @offset: The offset into the buffer's contents to start mapping
3165 * @len: Length of buffer space to be mapped
3166 *
3167 * Fill the specified scatter-gather list with mappings/pointers into a
3168 * region of the buffer space attached to a socket buffer.
3169 */
David S. Miller51c739d2007-10-30 21:29:29 -07003170static int
3171__skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len)
David Howells716ea3a2007-04-02 20:19:53 -07003172{
David S. Miller1a028e52007-04-27 15:21:23 -07003173 int start = skb_headlen(skb);
3174 int i, copy = start - offset;
David S. Millerfbb398a2009-06-09 00:18:59 -07003175 struct sk_buff *frag_iter;
David Howells716ea3a2007-04-02 20:19:53 -07003176 int elt = 0;
3177
3178 if (copy > 0) {
3179 if (copy > len)
3180 copy = len;
Jens Axboe642f1492007-10-24 11:20:47 +02003181 sg_set_buf(sg, skb->data + offset, copy);
David Howells716ea3a2007-04-02 20:19:53 -07003182 elt++;
3183 if ((len -= copy) == 0)
3184 return elt;
3185 offset += copy;
3186 }
3187
3188 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
David S. Miller1a028e52007-04-27 15:21:23 -07003189 int end;
David Howells716ea3a2007-04-02 20:19:53 -07003190
Ilpo Järvinen547b7922008-07-25 21:43:18 -07003191 WARN_ON(start > offset + len);
David S. Miller1a028e52007-04-27 15:21:23 -07003192
Eric Dumazet9e903e02011-10-18 21:00:24 +00003193 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
David Howells716ea3a2007-04-02 20:19:53 -07003194 if ((copy = end - offset) > 0) {
3195 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3196
3197 if (copy > len)
3198 copy = len;
Ian Campbellea2ab692011-08-22 23:44:58 +00003199 sg_set_page(&sg[elt], skb_frag_page(frag), copy,
Jens Axboe642f1492007-10-24 11:20:47 +02003200 frag->page_offset+offset-start);
David Howells716ea3a2007-04-02 20:19:53 -07003201 elt++;
3202 if (!(len -= copy))
3203 return elt;
3204 offset += copy;
3205 }
David S. Miller1a028e52007-04-27 15:21:23 -07003206 start = end;
David Howells716ea3a2007-04-02 20:19:53 -07003207 }
3208
David S. Millerfbb398a2009-06-09 00:18:59 -07003209 skb_walk_frags(skb, frag_iter) {
3210 int end;
David Howells716ea3a2007-04-02 20:19:53 -07003211
David S. Millerfbb398a2009-06-09 00:18:59 -07003212 WARN_ON(start > offset + len);
David Howells716ea3a2007-04-02 20:19:53 -07003213
David S. Millerfbb398a2009-06-09 00:18:59 -07003214 end = start + frag_iter->len;
3215 if ((copy = end - offset) > 0) {
3216 if (copy > len)
3217 copy = len;
3218 elt += __skb_to_sgvec(frag_iter, sg+elt, offset - start,
3219 copy);
3220 if ((len -= copy) == 0)
3221 return elt;
3222 offset += copy;
David Howells716ea3a2007-04-02 20:19:53 -07003223 }
David S. Millerfbb398a2009-06-09 00:18:59 -07003224 start = end;
David Howells716ea3a2007-04-02 20:19:53 -07003225 }
3226 BUG_ON(len);
3227 return elt;
3228}
3229
David S. Miller51c739d2007-10-30 21:29:29 -07003230int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len)
3231{
3232 int nsg = __skb_to_sgvec(skb, sg, offset, len);
3233
Jens Axboec46f2332007-10-31 12:06:37 +01003234 sg_mark_end(&sg[nsg - 1]);
David S. Miller51c739d2007-10-30 21:29:29 -07003235
3236 return nsg;
3237}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08003238EXPORT_SYMBOL_GPL(skb_to_sgvec);
David S. Miller51c739d2007-10-30 21:29:29 -07003239
David Howells716ea3a2007-04-02 20:19:53 -07003240/**
3241 * skb_cow_data - Check that a socket buffer's data buffers are writable
3242 * @skb: The socket buffer to check.
3243 * @tailbits: Amount of trailing space to be added
3244 * @trailer: Returned pointer to the skb where the @tailbits space begins
3245 *
3246 * Make sure that the data buffers attached to a socket buffer are
3247 * writable. If they are not, private copies are made of the data buffers
3248 * and the socket buffer is set to use these instead.
3249 *
3250 * If @tailbits is given, make sure that there is space to write @tailbits
3251 * bytes of data beyond current end of socket buffer. @trailer will be
3252 * set to point to the skb in which this space begins.
3253 *
3254 * The number of scatterlist elements required to completely map the
3255 * COW'd and extended socket buffer will be returned.
3256 */
3257int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer)
3258{
3259 int copyflag;
3260 int elt;
3261 struct sk_buff *skb1, **skb_p;
3262
3263 /* If skb is cloned or its head is paged, reallocate
3264 * head pulling out all the pages (pages are considered not writable
3265 * at the moment even if they are anonymous).
3266 */
3267 if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) &&
3268 __pskb_pull_tail(skb, skb_pagelen(skb)-skb_headlen(skb)) == NULL)
3269 return -ENOMEM;
3270
3271 /* Easy case. Most of packets will go this way. */
David S. Miller21dc3302010-08-23 00:13:46 -07003272 if (!skb_has_frag_list(skb)) {
David Howells716ea3a2007-04-02 20:19:53 -07003273 /* A little of trouble, not enough of space for trailer.
3274 * This should not happen, when stack is tuned to generate
3275 * good frames. OK, on miss we reallocate and reserve even more
3276 * space, 128 bytes is fair. */
3277
3278 if (skb_tailroom(skb) < tailbits &&
3279 pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC))
3280 return -ENOMEM;
3281
3282 /* Voila! */
3283 *trailer = skb;
3284 return 1;
3285 }
3286
3287 /* Misery. We are in troubles, going to mincer fragments... */
3288
3289 elt = 1;
3290 skb_p = &skb_shinfo(skb)->frag_list;
3291 copyflag = 0;
3292
3293 while ((skb1 = *skb_p) != NULL) {
3294 int ntail = 0;
3295
3296 /* The fragment is partially pulled by someone,
3297 * this can happen on input. Copy it and everything
3298 * after it. */
3299
3300 if (skb_shared(skb1))
3301 copyflag = 1;
3302
3303 /* If the skb is the last, worry about trailer. */
3304
3305 if (skb1->next == NULL && tailbits) {
3306 if (skb_shinfo(skb1)->nr_frags ||
David S. Miller21dc3302010-08-23 00:13:46 -07003307 skb_has_frag_list(skb1) ||
David Howells716ea3a2007-04-02 20:19:53 -07003308 skb_tailroom(skb1) < tailbits)
3309 ntail = tailbits + 128;
3310 }
3311
3312 if (copyflag ||
3313 skb_cloned(skb1) ||
3314 ntail ||
3315 skb_shinfo(skb1)->nr_frags ||
David S. Miller21dc3302010-08-23 00:13:46 -07003316 skb_has_frag_list(skb1)) {
David Howells716ea3a2007-04-02 20:19:53 -07003317 struct sk_buff *skb2;
3318
3319 /* Fuck, we are miserable poor guys... */
3320 if (ntail == 0)
3321 skb2 = skb_copy(skb1, GFP_ATOMIC);
3322 else
3323 skb2 = skb_copy_expand(skb1,
3324 skb_headroom(skb1),
3325 ntail,
3326 GFP_ATOMIC);
3327 if (unlikely(skb2 == NULL))
3328 return -ENOMEM;
3329
3330 if (skb1->sk)
3331 skb_set_owner_w(skb2, skb1->sk);
3332
3333 /* Looking around. Are we still alive?
3334 * OK, link new skb, drop old one */
3335
3336 skb2->next = skb1->next;
3337 *skb_p = skb2;
3338 kfree_skb(skb1);
3339 skb1 = skb2;
3340 }
3341 elt++;
3342 *trailer = skb1;
3343 skb_p = &skb1->next;
3344 }
3345
3346 return elt;
3347}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08003348EXPORT_SYMBOL_GPL(skb_cow_data);
David Howells716ea3a2007-04-02 20:19:53 -07003349
Eric Dumazetb1faf562010-05-31 23:44:05 -07003350static void sock_rmem_free(struct sk_buff *skb)
3351{
3352 struct sock *sk = skb->sk;
3353
3354 atomic_sub(skb->truesize, &sk->sk_rmem_alloc);
3355}
3356
3357/*
3358 * Note: We dont mem charge error packets (no sk_forward_alloc changes)
3359 */
3360int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb)
3361{
Eric Dumazet110c4332012-04-06 10:49:10 +02003362 int len = skb->len;
3363
Eric Dumazetb1faf562010-05-31 23:44:05 -07003364 if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >=
Eric Dumazet95c96172012-04-15 05:58:06 +00003365 (unsigned int)sk->sk_rcvbuf)
Eric Dumazetb1faf562010-05-31 23:44:05 -07003366 return -ENOMEM;
3367
3368 skb_orphan(skb);
3369 skb->sk = sk;
3370 skb->destructor = sock_rmem_free;
3371 atomic_add(skb->truesize, &sk->sk_rmem_alloc);
3372
Eric Dumazetabb57ea2011-05-18 02:21:31 -04003373 /* before exiting rcu section, make sure dst is refcounted */
3374 skb_dst_force(skb);
3375
Eric Dumazetb1faf562010-05-31 23:44:05 -07003376 skb_queue_tail(&sk->sk_error_queue, skb);
3377 if (!sock_flag(sk, SOCK_DEAD))
Eric Dumazet110c4332012-04-06 10:49:10 +02003378 sk->sk_data_ready(sk, len);
Eric Dumazetb1faf562010-05-31 23:44:05 -07003379 return 0;
3380}
3381EXPORT_SYMBOL(sock_queue_err_skb);
3382
Patrick Ohlyac45f602009-02-12 05:03:37 +00003383void skb_tstamp_tx(struct sk_buff *orig_skb,
3384 struct skb_shared_hwtstamps *hwtstamps)
3385{
3386 struct sock *sk = orig_skb->sk;
3387 struct sock_exterr_skb *serr;
3388 struct sk_buff *skb;
3389 int err;
3390
3391 if (!sk)
3392 return;
3393
Patrick Ohlyac45f602009-02-12 05:03:37 +00003394 if (hwtstamps) {
Willem de Bruijn2e313962013-04-23 00:39:28 +00003395 *skb_hwtstamps(orig_skb) =
Patrick Ohlyac45f602009-02-12 05:03:37 +00003396 *hwtstamps;
3397 } else {
3398 /*
3399 * no hardware time stamps available,
Oliver Hartkopp2244d072010-08-17 08:59:14 +00003400 * so keep the shared tx_flags and only
Patrick Ohlyac45f602009-02-12 05:03:37 +00003401 * store software time stamp
3402 */
Willem de Bruijn2e313962013-04-23 00:39:28 +00003403 orig_skb->tstamp = ktime_get_real();
Patrick Ohlyac45f602009-02-12 05:03:37 +00003404 }
3405
Willem de Bruijn2e313962013-04-23 00:39:28 +00003406 skb = skb_clone(orig_skb, GFP_ATOMIC);
3407 if (!skb)
3408 return;
3409
Patrick Ohlyac45f602009-02-12 05:03:37 +00003410 serr = SKB_EXT_ERR(skb);
3411 memset(serr, 0, sizeof(*serr));
3412 serr->ee.ee_errno = ENOMSG;
3413 serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING;
Eric Dumazet29030372010-05-29 00:20:48 -07003414
Patrick Ohlyac45f602009-02-12 05:03:37 +00003415 err = sock_queue_err_skb(sk, skb);
Eric Dumazet29030372010-05-29 00:20:48 -07003416
Patrick Ohlyac45f602009-02-12 05:03:37 +00003417 if (err)
3418 kfree_skb(skb);
3419}
3420EXPORT_SYMBOL_GPL(skb_tstamp_tx);
3421
Johannes Berg6e3e9392011-11-09 10:15:42 +01003422void skb_complete_wifi_ack(struct sk_buff *skb, bool acked)
3423{
3424 struct sock *sk = skb->sk;
3425 struct sock_exterr_skb *serr;
3426 int err;
3427
3428 skb->wifi_acked_valid = 1;
3429 skb->wifi_acked = acked;
3430
3431 serr = SKB_EXT_ERR(skb);
3432 memset(serr, 0, sizeof(*serr));
3433 serr->ee.ee_errno = ENOMSG;
3434 serr->ee.ee_origin = SO_EE_ORIGIN_TXSTATUS;
3435
3436 err = sock_queue_err_skb(sk, skb);
3437 if (err)
3438 kfree_skb(skb);
3439}
3440EXPORT_SYMBOL_GPL(skb_complete_wifi_ack);
3441
Patrick Ohlyac45f602009-02-12 05:03:37 +00003442
Rusty Russellf35d9d82008-02-04 23:49:54 -05003443/**
3444 * skb_partial_csum_set - set up and verify partial csum values for packet
3445 * @skb: the skb to set
3446 * @start: the number of bytes after skb->data to start checksumming.
3447 * @off: the offset from start to place the checksum.
3448 *
3449 * For untrusted partially-checksummed packets, we need to make sure the values
3450 * for skb->csum_start and skb->csum_offset are valid so we don't oops.
3451 *
3452 * This function checks and sets those values and skb->ip_summed: if this
3453 * returns false you should drop the packet.
3454 */
3455bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off)
3456{
Herbert Xu5ff8dda2009-06-04 01:22:01 +00003457 if (unlikely(start > skb_headlen(skb)) ||
3458 unlikely((int)start + off > skb_headlen(skb) - 2)) {
Joe Perchese87cc472012-05-13 21:56:26 +00003459 net_warn_ratelimited("bad partial csum: csum=%u/%u len=%u\n",
3460 start, off, skb_headlen(skb));
Rusty Russellf35d9d82008-02-04 23:49:54 -05003461 return false;
3462 }
3463 skb->ip_summed = CHECKSUM_PARTIAL;
3464 skb->csum_start = skb_headroom(skb) + start;
3465 skb->csum_offset = off;
Jason Wange5d5dec2013-03-26 23:11:20 +00003466 skb_set_transport_header(skb, start);
Rusty Russellf35d9d82008-02-04 23:49:54 -05003467 return true;
3468}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08003469EXPORT_SYMBOL_GPL(skb_partial_csum_set);
Rusty Russellf35d9d82008-02-04 23:49:54 -05003470
Ben Hutchings4497b072008-06-19 16:22:28 -07003471void __skb_warn_lro_forwarding(const struct sk_buff *skb)
3472{
Joe Perchese87cc472012-05-13 21:56:26 +00003473 net_warn_ratelimited("%s: received packets cannot be forwarded while LRO is enabled\n",
3474 skb->dev->name);
Ben Hutchings4497b072008-06-19 16:22:28 -07003475}
Ben Hutchings4497b072008-06-19 16:22:28 -07003476EXPORT_SYMBOL(__skb_warn_lro_forwarding);
Eric Dumazetbad43ca2012-05-19 03:02:02 +00003477
3478void kfree_skb_partial(struct sk_buff *skb, bool head_stolen)
3479{
Eric Dumazet3d861f62012-10-22 09:03:40 +00003480 if (head_stolen) {
3481 skb_release_head_state(skb);
Eric Dumazetbad43ca2012-05-19 03:02:02 +00003482 kmem_cache_free(skbuff_head_cache, skb);
Eric Dumazet3d861f62012-10-22 09:03:40 +00003483 } else {
Eric Dumazetbad43ca2012-05-19 03:02:02 +00003484 __kfree_skb(skb);
Eric Dumazet3d861f62012-10-22 09:03:40 +00003485 }
Eric Dumazetbad43ca2012-05-19 03:02:02 +00003486}
3487EXPORT_SYMBOL(kfree_skb_partial);
3488
3489/**
3490 * skb_try_coalesce - try to merge skb to prior one
3491 * @to: prior buffer
3492 * @from: buffer to add
3493 * @fragstolen: pointer to boolean
Randy Dunlapc6c4b972012-06-08 14:01:44 +00003494 * @delta_truesize: how much more was allocated than was requested
Eric Dumazetbad43ca2012-05-19 03:02:02 +00003495 */
3496bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
3497 bool *fragstolen, int *delta_truesize)
3498{
3499 int i, delta, len = from->len;
3500
3501 *fragstolen = false;
3502
3503 if (skb_cloned(to))
3504 return false;
3505
3506 if (len <= skb_tailroom(to)) {
3507 BUG_ON(skb_copy_bits(from, 0, skb_put(to, len), len));
3508 *delta_truesize = 0;
3509 return true;
3510 }
3511
3512 if (skb_has_frag_list(to) || skb_has_frag_list(from))
3513 return false;
3514
3515 if (skb_headlen(from) != 0) {
3516 struct page *page;
3517 unsigned int offset;
3518
3519 if (skb_shinfo(to)->nr_frags +
3520 skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS)
3521 return false;
3522
3523 if (skb_head_is_locked(from))
3524 return false;
3525
3526 delta = from->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff));
3527
3528 page = virt_to_head_page(from->head);
3529 offset = from->data - (unsigned char *)page_address(page);
3530
3531 skb_fill_page_desc(to, skb_shinfo(to)->nr_frags,
3532 page, offset, skb_headlen(from));
3533 *fragstolen = true;
3534 } else {
3535 if (skb_shinfo(to)->nr_frags +
3536 skb_shinfo(from)->nr_frags > MAX_SKB_FRAGS)
3537 return false;
3538
Weiping Panf4b549a2012-09-28 20:15:30 +00003539 delta = from->truesize - SKB_TRUESIZE(skb_end_offset(from));
Eric Dumazetbad43ca2012-05-19 03:02:02 +00003540 }
3541
3542 WARN_ON_ONCE(delta < len);
3543
3544 memcpy(skb_shinfo(to)->frags + skb_shinfo(to)->nr_frags,
3545 skb_shinfo(from)->frags,
3546 skb_shinfo(from)->nr_frags * sizeof(skb_frag_t));
3547 skb_shinfo(to)->nr_frags += skb_shinfo(from)->nr_frags;
3548
3549 if (!skb_cloned(from))
3550 skb_shinfo(from)->nr_frags = 0;
3551
Li RongQing8ea853f2012-09-18 16:53:21 +00003552 /* if the skb is not cloned this does nothing
3553 * since we set nr_frags to 0.
3554 */
Eric Dumazetbad43ca2012-05-19 03:02:02 +00003555 for (i = 0; i < skb_shinfo(from)->nr_frags; i++)
3556 skb_frag_ref(from, i);
3557
3558 to->truesize += delta;
3559 to->len += len;
3560 to->data_len += len;
3561
3562 *delta_truesize = delta;
3563 return true;
3564}
3565EXPORT_SYMBOL(skb_try_coalesce);
Nicolas Dichtel621e84d2013-06-26 16:11:27 +02003566
3567/**
Nicolas Dichtel8b27f272013-09-02 15:34:56 +02003568 * skb_scrub_packet - scrub an skb
Nicolas Dichtel621e84d2013-06-26 16:11:27 +02003569 *
3570 * @skb: buffer to clean
Nicolas Dichtel8b27f272013-09-02 15:34:56 +02003571 * @xnet: packet is crossing netns
Nicolas Dichtel621e84d2013-06-26 16:11:27 +02003572 *
Nicolas Dichtel8b27f272013-09-02 15:34:56 +02003573 * skb_scrub_packet can be used after encapsulating or decapsulting a packet
3574 * into/from a tunnel. Some information have to be cleared during these
3575 * operations.
3576 * skb_scrub_packet can also be used to clean a skb before injecting it in
3577 * another namespace (@xnet == true). We have to clear all information in the
3578 * skb that could impact namespace isolation.
Nicolas Dichtel621e84d2013-06-26 16:11:27 +02003579 */
Nicolas Dichtel8b27f272013-09-02 15:34:56 +02003580void skb_scrub_packet(struct sk_buff *skb, bool xnet)
Nicolas Dichtel621e84d2013-06-26 16:11:27 +02003581{
Nicolas Dichtel8b27f272013-09-02 15:34:56 +02003582 if (xnet)
3583 skb_orphan(skb);
Nicolas Dichtel621e84d2013-06-26 16:11:27 +02003584 skb->tstamp.tv64 = 0;
3585 skb->pkt_type = PACKET_HOST;
3586 skb->skb_iif = 0;
Hannes Frederic Sowa239c78d2013-12-05 23:29:19 +01003587 skb->local_df = 0;
Nicolas Dichtel621e84d2013-06-26 16:11:27 +02003588 skb_dst_drop(skb);
3589 skb->mark = 0;
3590 secpath_reset(skb);
3591 nf_reset(skb);
3592 nf_reset_trace(skb);
3593}
3594EXPORT_SYMBOL_GPL(skb_scrub_packet);