blob: 93c4e060c91e7bffb121d51eeccd264337214491 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Routines having to do with the 'struct sk_buff' memory handlers.
3 *
Alan Cox113aa832008-10-13 19:01:08 -07004 * Authors: Alan Cox <alan@lxorguk.ukuu.org.uk>
Linus Torvalds1da177e2005-04-16 15:20:36 -07005 * Florian La Roche <rzsfl@rz.uni-sb.de>
6 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07007 * Fixes:
8 * Alan Cox : Fixed the worst of the load
9 * balancer bugs.
10 * Dave Platt : Interrupt stacking fix.
11 * Richard Kooijman : Timestamp fixes.
12 * Alan Cox : Changed buffer format.
13 * Alan Cox : destructor hook for AF_UNIX etc.
14 * Linus Torvalds : Better skb_clone.
15 * Alan Cox : Added skb_copy.
16 * Alan Cox : Added all the changed routines Linus
17 * only put in the headers
18 * Ray VanTassle : Fixed --skb->lock in free
19 * Alan Cox : skb_copy copy arp field
20 * Andi Kleen : slabified it.
21 * Robert Olsson : Removed skb_head_pool
22 *
23 * NOTE:
24 * The __skb_ routines should be called with interrupts
25 * disabled, or you better be *real* sure that the operation is atomic
26 * with respect to whatever list is being frobbed (e.g. via lock_sock()
27 * or via disabling bottom half handlers, etc).
28 *
29 * This program is free software; you can redistribute it and/or
30 * modify it under the terms of the GNU General Public License
31 * as published by the Free Software Foundation; either version
32 * 2 of the License, or (at your option) any later version.
33 */
34
35/*
36 * The functions in this file will not compile correctly with gcc 2.4.x
37 */
38
Linus Torvalds1da177e2005-04-16 15:20:36 -070039#include <linux/module.h>
40#include <linux/types.h>
41#include <linux/kernel.h>
Vegard Nossumfe55f6d2008-08-30 12:16:35 +020042#include <linux/kmemcheck.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070043#include <linux/mm.h>
44#include <linux/interrupt.h>
45#include <linux/in.h>
46#include <linux/inet.h>
47#include <linux/slab.h>
48#include <linux/netdevice.h>
49#ifdef CONFIG_NET_CLS_ACT
50#include <net/pkt_sched.h>
51#endif
52#include <linux/string.h>
53#include <linux/skbuff.h>
Jens Axboe9c55e012007-11-06 23:30:13 -080054#include <linux/splice.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070055#include <linux/cache.h>
56#include <linux/rtnetlink.h>
57#include <linux/init.h>
David Howells716ea3a2007-04-02 20:19:53 -070058#include <linux/scatterlist.h>
Patrick Ohlyac45f602009-02-12 05:03:37 +000059#include <linux/errqueue.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070060
61#include <net/protocol.h>
62#include <net/dst.h>
63#include <net/sock.h>
64#include <net/checksum.h>
65#include <net/xfrm.h>
66
67#include <asm/uaccess.h>
68#include <asm/system.h>
Steven Rostedtad8d75f2009-04-14 19:39:12 -040069#include <trace/events/skb.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070070
Al Viroa1f8e7f72006-10-19 16:08:53 -040071#include "kmap_skb.h"
72
Christoph Lametere18b8902006-12-06 20:33:20 -080073static struct kmem_cache *skbuff_head_cache __read_mostly;
74static struct kmem_cache *skbuff_fclone_cache __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -070075
Jens Axboe9c55e012007-11-06 23:30:13 -080076static void sock_pipe_buf_release(struct pipe_inode_info *pipe,
77 struct pipe_buffer *buf)
78{
Jarek Poplawski8b9d3722009-01-19 17:03:56 -080079 put_page(buf->page);
Jens Axboe9c55e012007-11-06 23:30:13 -080080}
81
82static void sock_pipe_buf_get(struct pipe_inode_info *pipe,
83 struct pipe_buffer *buf)
84{
Jarek Poplawski8b9d3722009-01-19 17:03:56 -080085 get_page(buf->page);
Jens Axboe9c55e012007-11-06 23:30:13 -080086}
87
88static int sock_pipe_buf_steal(struct pipe_inode_info *pipe,
89 struct pipe_buffer *buf)
90{
91 return 1;
92}
93
94
95/* Pipe buffer operations for a socket. */
Alexey Dobriyan28dfef82009-12-15 16:46:48 -080096static const struct pipe_buf_operations sock_pipe_buf_ops = {
Jens Axboe9c55e012007-11-06 23:30:13 -080097 .can_merge = 0,
98 .map = generic_pipe_buf_map,
99 .unmap = generic_pipe_buf_unmap,
100 .confirm = generic_pipe_buf_confirm,
101 .release = sock_pipe_buf_release,
102 .steal = sock_pipe_buf_steal,
103 .get = sock_pipe_buf_get,
104};
105
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106/*
107 * Keep out-of-line to prevent kernel bloat.
108 * __builtin_return_address is not used because it is not always
109 * reliable.
110 */
111
112/**
113 * skb_over_panic - private function
114 * @skb: buffer
115 * @sz: size
116 * @here: address
117 *
118 * Out of line support code for skb_put(). Not user callable.
119 */
120void skb_over_panic(struct sk_buff *skb, int sz, void *here)
121{
Patrick McHardy26095452005-04-21 16:43:02 -0700122 printk(KERN_EMERG "skb_over_panic: text:%p len:%d put:%d head:%p "
Arnaldo Carvalho de Melo4305b542007-04-19 20:43:29 -0700123 "data:%p tail:%#lx end:%#lx dev:%s\n",
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -0700124 here, skb->len, sz, skb->head, skb->data,
Arnaldo Carvalho de Melo4305b542007-04-19 20:43:29 -0700125 (unsigned long)skb->tail, (unsigned long)skb->end,
Patrick McHardy26095452005-04-21 16:43:02 -0700126 skb->dev ? skb->dev->name : "<NULL>");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700127 BUG();
128}
David S. Millerb4ac530fc2009-02-10 02:09:24 -0800129EXPORT_SYMBOL(skb_over_panic);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700130
131/**
132 * skb_under_panic - private function
133 * @skb: buffer
134 * @sz: size
135 * @here: address
136 *
137 * Out of line support code for skb_push(). Not user callable.
138 */
139
140void skb_under_panic(struct sk_buff *skb, int sz, void *here)
141{
Patrick McHardy26095452005-04-21 16:43:02 -0700142 printk(KERN_EMERG "skb_under_panic: text:%p len:%d put:%d head:%p "
Arnaldo Carvalho de Melo4305b542007-04-19 20:43:29 -0700143 "data:%p tail:%#lx end:%#lx dev:%s\n",
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -0700144 here, skb->len, sz, skb->head, skb->data,
Arnaldo Carvalho de Melo4305b542007-04-19 20:43:29 -0700145 (unsigned long)skb->tail, (unsigned long)skb->end,
Patrick McHardy26095452005-04-21 16:43:02 -0700146 skb->dev ? skb->dev->name : "<NULL>");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700147 BUG();
148}
David S. Millerb4ac530fc2009-02-10 02:09:24 -0800149EXPORT_SYMBOL(skb_under_panic);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700150
151/* Allocate a new skbuff. We do this ourselves so we can fill in a few
152 * 'private' fields and also do memory statistics to find all the
153 * [BEEP] leaks.
154 *
155 */
156
157/**
David S. Millerd179cd12005-08-17 14:57:30 -0700158 * __alloc_skb - allocate a network buffer
Linus Torvalds1da177e2005-04-16 15:20:36 -0700159 * @size: size to allocate
160 * @gfp_mask: allocation mask
Randy Dunlapc83c2482005-10-18 22:07:41 -0700161 * @fclone: allocate from fclone cache instead of head cache
162 * and allocate a cloned (child) skb
Christoph Hellwigb30973f2006-12-06 20:32:36 -0800163 * @node: numa node to allocate memory on
Linus Torvalds1da177e2005-04-16 15:20:36 -0700164 *
165 * Allocate a new &sk_buff. The returned buffer has no headroom and a
166 * tail room of size bytes. The object has a reference count of one.
167 * The return is the buffer. On a failure the return is %NULL.
168 *
169 * Buffers may only be allocated from interrupts using a @gfp_mask of
170 * %GFP_ATOMIC.
171 */
Al Virodd0fc662005-10-07 07:46:04 +0100172struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
Christoph Hellwigb30973f2006-12-06 20:32:36 -0800173 int fclone, int node)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700174{
Christoph Lametere18b8902006-12-06 20:33:20 -0800175 struct kmem_cache *cache;
Benjamin LaHaise4947d3e2006-01-03 14:06:50 -0800176 struct skb_shared_info *shinfo;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700177 struct sk_buff *skb;
178 u8 *data;
179
Herbert Xu8798b3f2006-01-23 16:32:45 -0800180 cache = fclone ? skbuff_fclone_cache : skbuff_head_cache;
181
Linus Torvalds1da177e2005-04-16 15:20:36 -0700182 /* Get the HEAD */
Christoph Hellwigb30973f2006-12-06 20:32:36 -0800183 skb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184 if (!skb)
185 goto out;
186
Linus Torvalds1da177e2005-04-16 15:20:36 -0700187 size = SKB_DATA_ALIGN(size);
Christoph Hellwigb30973f2006-12-06 20:32:36 -0800188 data = kmalloc_node_track_caller(size + sizeof(struct skb_shared_info),
189 gfp_mask, node);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700190 if (!data)
191 goto nodata;
192
Arnaldo Carvalho de Meloca0605a2007-03-19 10:48:59 -0300193 /*
Johannes Bergc8005782008-05-03 20:56:42 -0700194 * Only clear those fields we need to clear, not those that we will
195 * actually initialise below. Hence, don't put any more fields after
196 * the tail pointer in struct sk_buff!
Arnaldo Carvalho de Meloca0605a2007-03-19 10:48:59 -0300197 */
198 memset(skb, 0, offsetof(struct sk_buff, tail));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700199 skb->truesize = size + sizeof(struct sk_buff);
200 atomic_set(&skb->users, 1);
201 skb->head = data;
202 skb->data = data;
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -0700203 skb_reset_tail_pointer(skb);
Arnaldo Carvalho de Melo4305b542007-04-19 20:43:29 -0700204 skb->end = skb->tail + size;
Vegard Nossumfe55f6d2008-08-30 12:16:35 +0200205 kmemcheck_annotate_bitfield(skb, flags1);
206 kmemcheck_annotate_bitfield(skb, flags2);
Stephen Hemminger19633e12009-06-17 05:23:27 +0000207#ifdef NET_SKBUFF_DATA_USES_OFFSET
208 skb->mac_header = ~0U;
209#endif
210
Benjamin LaHaise4947d3e2006-01-03 14:06:50 -0800211 /* make sure we initialize shinfo sequentially */
212 shinfo = skb_shinfo(skb);
213 atomic_set(&shinfo->dataref, 1);
214 shinfo->nr_frags = 0;
Herbert Xu79671682006-06-22 02:40:14 -0700215 shinfo->gso_size = 0;
216 shinfo->gso_segs = 0;
217 shinfo->gso_type = 0;
Benjamin LaHaise4947d3e2006-01-03 14:06:50 -0800218 shinfo->ip6_frag_id = 0;
Patrick Ohlyac45f602009-02-12 05:03:37 +0000219 shinfo->tx_flags.flags = 0;
David S. Millerfbb398a2009-06-09 00:18:59 -0700220 skb_frag_list_init(skb);
Patrick Ohlyac45f602009-02-12 05:03:37 +0000221 memset(&shinfo->hwtstamps, 0, sizeof(shinfo->hwtstamps));
Benjamin LaHaise4947d3e2006-01-03 14:06:50 -0800222
David S. Millerd179cd12005-08-17 14:57:30 -0700223 if (fclone) {
224 struct sk_buff *child = skb + 1;
225 atomic_t *fclone_ref = (atomic_t *) (child + 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700226
Vegard Nossumfe55f6d2008-08-30 12:16:35 +0200227 kmemcheck_annotate_bitfield(child, flags1);
228 kmemcheck_annotate_bitfield(child, flags2);
David S. Millerd179cd12005-08-17 14:57:30 -0700229 skb->fclone = SKB_FCLONE_ORIG;
230 atomic_set(fclone_ref, 1);
231
232 child->fclone = SKB_FCLONE_UNAVAILABLE;
233 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700234out:
235 return skb;
236nodata:
Herbert Xu8798b3f2006-01-23 16:32:45 -0800237 kmem_cache_free(cache, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700238 skb = NULL;
239 goto out;
240}
David S. Millerb4ac530fc2009-02-10 02:09:24 -0800241EXPORT_SYMBOL(__alloc_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700242
243/**
Christoph Hellwig8af27452006-07-31 22:35:23 -0700244 * __netdev_alloc_skb - allocate an skbuff for rx on a specific device
245 * @dev: network device to receive on
246 * @length: length to allocate
247 * @gfp_mask: get_free_pages mask, passed to alloc_skb
248 *
249 * Allocate a new &sk_buff and assign it a usage count of one. The
250 * buffer has unspecified headroom built in. Users should allocate
251 * the headroom they think they need without accounting for the
252 * built in space. The built in space is used for optimisations.
253 *
254 * %NULL is returned if there is no free memory.
255 */
256struct sk_buff *__netdev_alloc_skb(struct net_device *dev,
257 unsigned int length, gfp_t gfp_mask)
258{
Greg Kroah-Hartman43cb76d2002-04-09 12:14:34 -0700259 int node = dev->dev.parent ? dev_to_node(dev->dev.parent) : -1;
Christoph Hellwig8af27452006-07-31 22:35:23 -0700260 struct sk_buff *skb;
261
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900262 skb = __alloc_skb(length + NET_SKB_PAD, gfp_mask, 0, node);
Christoph Hellwig7b2e4972006-08-07 16:09:04 -0700263 if (likely(skb)) {
Christoph Hellwig8af27452006-07-31 22:35:23 -0700264 skb_reserve(skb, NET_SKB_PAD);
Christoph Hellwig7b2e4972006-08-07 16:09:04 -0700265 skb->dev = dev;
266 }
Christoph Hellwig8af27452006-07-31 22:35:23 -0700267 return skb;
268}
David S. Millerb4ac530fc2009-02-10 02:09:24 -0800269EXPORT_SYMBOL(__netdev_alloc_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700270
Peter Zijlstra654bed12008-10-07 14:22:33 -0700271struct page *__netdev_alloc_page(struct net_device *dev, gfp_t gfp_mask)
272{
273 int node = dev->dev.parent ? dev_to_node(dev->dev.parent) : -1;
274 struct page *page;
275
276 page = alloc_pages_node(node, gfp_mask, 0);
277 return page;
278}
279EXPORT_SYMBOL(__netdev_alloc_page);
280
281void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off,
282 int size)
283{
284 skb_fill_page_desc(skb, i, page, off, size);
285 skb->len += size;
286 skb->data_len += size;
287 skb->truesize += size;
288}
289EXPORT_SYMBOL(skb_add_rx_frag);
290
Ilpo Järvinenf58518e2008-03-27 17:51:31 -0700291/**
292 * dev_alloc_skb - allocate an skbuff for receiving
293 * @length: length to allocate
294 *
295 * Allocate a new &sk_buff and assign it a usage count of one. The
296 * buffer has unspecified headroom built in. Users should allocate
297 * the headroom they think they need without accounting for the
298 * built in space. The built in space is used for optimisations.
299 *
300 * %NULL is returned if there is no free memory. Although this function
301 * allocates memory it can be called from an interrupt.
302 */
303struct sk_buff *dev_alloc_skb(unsigned int length)
304{
Denys Vlasenko1483b872008-03-28 15:57:39 -0700305 /*
306 * There is more code here than it seems:
David S. Millera0f55e02008-03-28 18:22:32 -0700307 * __dev_alloc_skb is an inline
Denys Vlasenko1483b872008-03-28 15:57:39 -0700308 */
Ilpo Järvinenf58518e2008-03-27 17:51:31 -0700309 return __dev_alloc_skb(length, GFP_ATOMIC);
310}
311EXPORT_SYMBOL(dev_alloc_skb);
312
Herbert Xu27b437c2006-07-13 19:26:39 -0700313static void skb_drop_list(struct sk_buff **listp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700314{
Herbert Xu27b437c2006-07-13 19:26:39 -0700315 struct sk_buff *list = *listp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700316
Herbert Xu27b437c2006-07-13 19:26:39 -0700317 *listp = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700318
319 do {
320 struct sk_buff *this = list;
321 list = list->next;
322 kfree_skb(this);
323 } while (list);
324}
325
Herbert Xu27b437c2006-07-13 19:26:39 -0700326static inline void skb_drop_fraglist(struct sk_buff *skb)
327{
328 skb_drop_list(&skb_shinfo(skb)->frag_list);
329}
330
Linus Torvalds1da177e2005-04-16 15:20:36 -0700331static void skb_clone_fraglist(struct sk_buff *skb)
332{
333 struct sk_buff *list;
334
David S. Millerfbb398a2009-06-09 00:18:59 -0700335 skb_walk_frags(skb, list)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700336 skb_get(list);
337}
338
Adrian Bunk5bba1712006-06-29 13:02:35 -0700339static void skb_release_data(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700340{
341 if (!skb->cloned ||
342 !atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1,
343 &skb_shinfo(skb)->dataref)) {
344 if (skb_shinfo(skb)->nr_frags) {
345 int i;
346 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
347 put_page(skb_shinfo(skb)->frags[i].page);
348 }
349
David S. Millerfbb398a2009-06-09 00:18:59 -0700350 if (skb_has_frags(skb))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700351 skb_drop_fraglist(skb);
352
353 kfree(skb->head);
354 }
355}
356
357/*
358 * Free an skbuff by memory without cleaning the state.
359 */
Herbert Xu2d4baff2007-11-26 23:11:19 +0800360static void kfree_skbmem(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700361{
David S. Millerd179cd12005-08-17 14:57:30 -0700362 struct sk_buff *other;
363 atomic_t *fclone_ref;
364
David S. Millerd179cd12005-08-17 14:57:30 -0700365 switch (skb->fclone) {
366 case SKB_FCLONE_UNAVAILABLE:
367 kmem_cache_free(skbuff_head_cache, skb);
368 break;
369
370 case SKB_FCLONE_ORIG:
371 fclone_ref = (atomic_t *) (skb + 2);
372 if (atomic_dec_and_test(fclone_ref))
373 kmem_cache_free(skbuff_fclone_cache, skb);
374 break;
375
376 case SKB_FCLONE_CLONE:
377 fclone_ref = (atomic_t *) (skb + 1);
378 other = skb - 1;
379
380 /* The clone portion is available for
381 * fast-cloning again.
382 */
383 skb->fclone = SKB_FCLONE_UNAVAILABLE;
384
385 if (atomic_dec_and_test(fclone_ref))
386 kmem_cache_free(skbuff_fclone_cache, other);
387 break;
Stephen Hemminger3ff50b72007-04-20 17:09:22 -0700388 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700389}
390
Lennert Buytenhek04a4bb52008-10-01 02:33:12 -0700391static void skb_release_head_state(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700392{
Eric Dumazetadf30902009-06-02 05:19:30 +0000393 skb_dst_drop(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700394#ifdef CONFIG_XFRM
395 secpath_put(skb->sp);
396#endif
Stephen Hemminger9c2b3322005-04-19 22:39:42 -0700397 if (skb->destructor) {
398 WARN_ON(in_irq());
Linus Torvalds1da177e2005-04-16 15:20:36 -0700399 skb->destructor(skb);
400 }
Yasuyuki Kozakai9fb9cbb2005-11-09 16:38:16 -0800401#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
Yasuyuki Kozakai5f79e0f2007-03-23 11:17:07 -0700402 nf_conntrack_put(skb->nfct);
Yasuyuki Kozakai9fb9cbb2005-11-09 16:38:16 -0800403 nf_conntrack_put_reasm(skb->nfct_reasm);
404#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700405#ifdef CONFIG_BRIDGE_NETFILTER
406 nf_bridge_put(skb->nf_bridge);
407#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700408/* XXX: IS this still necessary? - JHS */
409#ifdef CONFIG_NET_SCHED
410 skb->tc_index = 0;
411#ifdef CONFIG_NET_CLS_ACT
412 skb->tc_verd = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700413#endif
414#endif
Lennert Buytenhek04a4bb52008-10-01 02:33:12 -0700415}
416
417/* Free everything but the sk_buff shell. */
418static void skb_release_all(struct sk_buff *skb)
419{
420 skb_release_head_state(skb);
Herbert Xu2d4baff2007-11-26 23:11:19 +0800421 skb_release_data(skb);
422}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700423
Herbert Xu2d4baff2007-11-26 23:11:19 +0800424/**
425 * __kfree_skb - private function
426 * @skb: buffer
427 *
428 * Free an sk_buff. Release anything attached to the buffer.
429 * Clean the state. This is an internal helper function. Users should
430 * always call kfree_skb
431 */
432
433void __kfree_skb(struct sk_buff *skb)
434{
435 skb_release_all(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700436 kfree_skbmem(skb);
437}
David S. Millerb4ac530fc2009-02-10 02:09:24 -0800438EXPORT_SYMBOL(__kfree_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700439
440/**
Jörn Engel231d06a2006-03-20 21:28:35 -0800441 * kfree_skb - free an sk_buff
442 * @skb: buffer to free
443 *
444 * Drop a reference to the buffer and free it if the usage count has
445 * hit zero.
446 */
447void kfree_skb(struct sk_buff *skb)
448{
449 if (unlikely(!skb))
450 return;
451 if (likely(atomic_read(&skb->users) == 1))
452 smp_rmb();
453 else if (likely(!atomic_dec_and_test(&skb->users)))
454 return;
Neil Hormanead2ceb2009-03-11 09:49:55 +0000455 trace_kfree_skb(skb, __builtin_return_address(0));
Jörn Engel231d06a2006-03-20 21:28:35 -0800456 __kfree_skb(skb);
457}
David S. Millerb4ac530fc2009-02-10 02:09:24 -0800458EXPORT_SYMBOL(kfree_skb);
Jörn Engel231d06a2006-03-20 21:28:35 -0800459
Stephen Hemmingerd1a203e2008-11-01 21:01:09 -0700460/**
Neil Hormanead2ceb2009-03-11 09:49:55 +0000461 * consume_skb - free an skbuff
462 * @skb: buffer to free
463 *
464 * Drop a ref to the buffer and free it if the usage count has hit zero
465 * Functions identically to kfree_skb, but kfree_skb assumes that the frame
466 * is being dropped after a failure and notes that
467 */
468void consume_skb(struct sk_buff *skb)
469{
470 if (unlikely(!skb))
471 return;
472 if (likely(atomic_read(&skb->users) == 1))
473 smp_rmb();
474 else if (likely(!atomic_dec_and_test(&skb->users)))
475 return;
476 __kfree_skb(skb);
477}
478EXPORT_SYMBOL(consume_skb);
479
480/**
Stephen Hemmingerd1a203e2008-11-01 21:01:09 -0700481 * skb_recycle_check - check if skb can be reused for receive
482 * @skb: buffer
483 * @skb_size: minimum receive buffer size
484 *
485 * Checks that the skb passed in is not shared or cloned, and
486 * that it is linear and its head portion at least as large as
487 * skb_size so that it can be recycled as a receive buffer.
488 * If these conditions are met, this function does any necessary
489 * reference count dropping and cleans up the skbuff as if it
490 * just came from __alloc_skb().
491 */
Lennert Buytenhek04a4bb52008-10-01 02:33:12 -0700492int skb_recycle_check(struct sk_buff *skb, int skb_size)
493{
494 struct skb_shared_info *shinfo;
495
Anton Vorontsove84af6d2009-11-10 14:11:01 +0000496 if (irqs_disabled())
497 return 0;
498
Lennert Buytenhek04a4bb52008-10-01 02:33:12 -0700499 if (skb_is_nonlinear(skb) || skb->fclone != SKB_FCLONE_UNAVAILABLE)
500 return 0;
501
502 skb_size = SKB_DATA_ALIGN(skb_size + NET_SKB_PAD);
503 if (skb_end_pointer(skb) - skb->head < skb_size)
504 return 0;
505
506 if (skb_shared(skb) || skb_cloned(skb))
507 return 0;
508
509 skb_release_head_state(skb);
510 shinfo = skb_shinfo(skb);
511 atomic_set(&shinfo->dataref, 1);
512 shinfo->nr_frags = 0;
513 shinfo->gso_size = 0;
514 shinfo->gso_segs = 0;
515 shinfo->gso_type = 0;
516 shinfo->ip6_frag_id = 0;
Lennert Buytenhekb8050072009-05-06 16:49:18 -0700517 shinfo->tx_flags.flags = 0;
David S. Millerfbb398a2009-06-09 00:18:59 -0700518 skb_frag_list_init(skb);
Lennert Buytenhekb8050072009-05-06 16:49:18 -0700519 memset(&shinfo->hwtstamps, 0, sizeof(shinfo->hwtstamps));
Lennert Buytenhek04a4bb52008-10-01 02:33:12 -0700520
521 memset(skb, 0, offsetof(struct sk_buff, tail));
Lennert Buytenhek04a4bb52008-10-01 02:33:12 -0700522 skb->data = skb->head + NET_SKB_PAD;
Lennert Buytenhek5cd33db2008-11-10 21:45:05 -0800523 skb_reset_tail_pointer(skb);
Lennert Buytenhek04a4bb52008-10-01 02:33:12 -0700524
525 return 1;
526}
527EXPORT_SYMBOL(skb_recycle_check);
528
Herbert Xudec18812007-10-14 00:37:30 -0700529static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
530{
531 new->tstamp = old->tstamp;
532 new->dev = old->dev;
533 new->transport_header = old->transport_header;
534 new->network_header = old->network_header;
535 new->mac_header = old->mac_header;
Eric Dumazetadf30902009-06-02 05:19:30 +0000536 skb_dst_set(new, dst_clone(skb_dst(old)));
Alexey Dobriyandef8b4f2008-10-28 13:24:06 -0700537#ifdef CONFIG_XFRM
Herbert Xudec18812007-10-14 00:37:30 -0700538 new->sp = secpath_get(old->sp);
539#endif
540 memcpy(new->cb, old->cb, sizeof(old->cb));
Herbert Xu9bcb97c2009-05-22 22:20:02 +0000541 new->csum = old->csum;
Herbert Xudec18812007-10-14 00:37:30 -0700542 new->local_df = old->local_df;
543 new->pkt_type = old->pkt_type;
544 new->ip_summed = old->ip_summed;
545 skb_copy_queue_mapping(new, old);
546 new->priority = old->priority;
547#if defined(CONFIG_IP_VS) || defined(CONFIG_IP_VS_MODULE)
548 new->ipvs_property = old->ipvs_property;
549#endif
550 new->protocol = old->protocol;
551 new->mark = old->mark;
Eric Dumazet8964be42009-11-20 15:35:04 -0800552 new->skb_iif = old->skb_iif;
Herbert Xudec18812007-10-14 00:37:30 -0700553 __nf_copy(new, old);
554#if defined(CONFIG_NETFILTER_XT_TARGET_TRACE) || \
555 defined(CONFIG_NETFILTER_XT_TARGET_TRACE_MODULE)
556 new->nf_trace = old->nf_trace;
557#endif
558#ifdef CONFIG_NET_SCHED
559 new->tc_index = old->tc_index;
560#ifdef CONFIG_NET_CLS_ACT
561 new->tc_verd = old->tc_verd;
562#endif
563#endif
Patrick McHardy6aa895b02008-07-14 22:49:06 -0700564 new->vlan_tci = old->vlan_tci;
565
Herbert Xudec18812007-10-14 00:37:30 -0700566 skb_copy_secmark(new, old);
567}
568
Herbert Xu82c49a32009-05-22 22:11:37 +0000569/*
570 * You should not add any new code to this function. Add it to
571 * __copy_skb_header above instead.
572 */
Herbert Xue0053ec2007-10-14 00:37:52 -0700573static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700574{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700575#define C(x) n->x = skb->x
576
577 n->next = n->prev = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700578 n->sk = NULL;
Herbert Xudec18812007-10-14 00:37:30 -0700579 __copy_skb_header(n, skb);
580
Linus Torvalds1da177e2005-04-16 15:20:36 -0700581 C(len);
582 C(data_len);
Alexey Dobriyan3e6b3b22007-03-16 15:00:46 -0700583 C(mac_len);
Patrick McHardy334a8132007-06-25 04:35:20 -0700584 n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len;
Paul Moore02f1c892008-01-07 21:56:41 -0800585 n->cloned = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700586 n->nohdr = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700587 n->destructor = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700588 C(tail);
589 C(end);
Paul Moore02f1c892008-01-07 21:56:41 -0800590 C(head);
591 C(data);
592 C(truesize);
593 atomic_set(&n->users, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700594
595 atomic_inc(&(skb_shinfo(skb)->dataref));
596 skb->cloned = 1;
597
598 return n;
Herbert Xue0053ec2007-10-14 00:37:52 -0700599#undef C
600}
601
602/**
603 * skb_morph - morph one skb into another
604 * @dst: the skb to receive the contents
605 * @src: the skb to supply the contents
606 *
607 * This is identical to skb_clone except that the target skb is
608 * supplied by the user.
609 *
610 * The target skb is returned upon exit.
611 */
612struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src)
613{
Herbert Xu2d4baff2007-11-26 23:11:19 +0800614 skb_release_all(dst);
Herbert Xue0053ec2007-10-14 00:37:52 -0700615 return __skb_clone(dst, src);
616}
617EXPORT_SYMBOL_GPL(skb_morph);
618
619/**
620 * skb_clone - duplicate an sk_buff
621 * @skb: buffer to clone
622 * @gfp_mask: allocation priority
623 *
624 * Duplicate an &sk_buff. The new one is not owned by a socket. Both
625 * copies share the same packet data but not structure. The new
626 * buffer has a reference count of 1. If the allocation fails the
627 * function returns %NULL otherwise the new buffer is returned.
628 *
629 * If this function is called from an interrupt gfp_mask() must be
630 * %GFP_ATOMIC.
631 */
632
633struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask)
634{
635 struct sk_buff *n;
636
637 n = skb + 1;
638 if (skb->fclone == SKB_FCLONE_ORIG &&
639 n->fclone == SKB_FCLONE_UNAVAILABLE) {
640 atomic_t *fclone_ref = (atomic_t *) (n + 1);
641 n->fclone = SKB_FCLONE_CLONE;
642 atomic_inc(fclone_ref);
643 } else {
644 n = kmem_cache_alloc(skbuff_head_cache, gfp_mask);
645 if (!n)
646 return NULL;
Vegard Nossumfe55f6d2008-08-30 12:16:35 +0200647
648 kmemcheck_annotate_bitfield(n, flags1);
649 kmemcheck_annotate_bitfield(n, flags2);
Herbert Xue0053ec2007-10-14 00:37:52 -0700650 n->fclone = SKB_FCLONE_UNAVAILABLE;
651 }
652
653 return __skb_clone(n, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700654}
David S. Millerb4ac530fc2009-02-10 02:09:24 -0800655EXPORT_SYMBOL(skb_clone);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700656
657static void copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
658{
Arnaldo Carvalho de Melo2e07fa92007-04-10 21:22:35 -0700659#ifndef NET_SKBUFF_DATA_USES_OFFSET
Linus Torvalds1da177e2005-04-16 15:20:36 -0700660 /*
661 * Shift between the two data areas in bytes
662 */
663 unsigned long offset = new->data - old->data;
Arnaldo Carvalho de Melo2e07fa92007-04-10 21:22:35 -0700664#endif
Herbert Xudec18812007-10-14 00:37:30 -0700665
666 __copy_skb_header(new, old);
667
Arnaldo Carvalho de Melo2e07fa92007-04-10 21:22:35 -0700668#ifndef NET_SKBUFF_DATA_USES_OFFSET
669 /* {transport,network,mac}_header are relative to skb->head */
670 new->transport_header += offset;
671 new->network_header += offset;
Stephen Hemminger603a8bb2009-06-17 12:17:34 +0000672 if (skb_mac_header_was_set(new))
673 new->mac_header += offset;
Arnaldo Carvalho de Melo2e07fa92007-04-10 21:22:35 -0700674#endif
Herbert Xu79671682006-06-22 02:40:14 -0700675 skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size;
676 skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs;
677 skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700678}
679
680/**
681 * skb_copy - create private copy of an sk_buff
682 * @skb: buffer to copy
683 * @gfp_mask: allocation priority
684 *
685 * Make a copy of both an &sk_buff and its data. This is used when the
686 * caller wishes to modify the data and needs a private copy of the
687 * data to alter. Returns %NULL on failure or the pointer to the buffer
688 * on success. The returned buffer has a reference count of 1.
689 *
690 * As by-product this function converts non-linear &sk_buff to linear
691 * one, so that &sk_buff becomes completely private and caller is allowed
692 * to modify all the data of returned buffer. This means that this
693 * function is not recommended for use in circumstances when only
694 * header is going to be modified. Use pskb_copy() instead.
695 */
696
Al Virodd0fc662005-10-07 07:46:04 +0100697struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700698{
699 int headerlen = skb->data - skb->head;
700 /*
701 * Allocate the copy buffer
702 */
Arnaldo Carvalho de Melo4305b542007-04-19 20:43:29 -0700703 struct sk_buff *n;
704#ifdef NET_SKBUFF_DATA_USES_OFFSET
705 n = alloc_skb(skb->end + skb->data_len, gfp_mask);
706#else
707 n = alloc_skb(skb->end - skb->head + skb->data_len, gfp_mask);
708#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700709 if (!n)
710 return NULL;
711
712 /* Set the data pointer */
713 skb_reserve(n, headerlen);
714 /* Set the tail pointer and length */
715 skb_put(n, skb->len);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700716
717 if (skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len))
718 BUG();
719
720 copy_skb_header(n, skb);
721 return n;
722}
David S. Millerb4ac530fc2009-02-10 02:09:24 -0800723EXPORT_SYMBOL(skb_copy);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700724
725/**
726 * pskb_copy - create copy of an sk_buff with private head.
727 * @skb: buffer to copy
728 * @gfp_mask: allocation priority
729 *
730 * Make a copy of both an &sk_buff and part of its data, located
731 * in header. Fragmented data remain shared. This is used when
732 * the caller wishes to modify only header of &sk_buff and needs
733 * private copy of the header to alter. Returns %NULL on failure
734 * or the pointer to the buffer on success.
735 * The returned buffer has a reference count of 1.
736 */
737
Al Virodd0fc662005-10-07 07:46:04 +0100738struct sk_buff *pskb_copy(struct sk_buff *skb, gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700739{
740 /*
741 * Allocate the copy buffer
742 */
Arnaldo Carvalho de Melo4305b542007-04-19 20:43:29 -0700743 struct sk_buff *n;
744#ifdef NET_SKBUFF_DATA_USES_OFFSET
745 n = alloc_skb(skb->end, gfp_mask);
746#else
747 n = alloc_skb(skb->end - skb->head, gfp_mask);
748#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700749 if (!n)
750 goto out;
751
752 /* Set the data pointer */
753 skb_reserve(n, skb->data - skb->head);
754 /* Set the tail pointer and length */
755 skb_put(n, skb_headlen(skb));
756 /* Copy the bytes */
Arnaldo Carvalho de Melod626f622007-03-27 18:55:52 -0300757 skb_copy_from_linear_data(skb, n->data, n->len);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700758
Herbert Xu25f484a2006-11-07 14:57:15 -0800759 n->truesize += skb->data_len;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700760 n->data_len = skb->data_len;
761 n->len = skb->len;
762
763 if (skb_shinfo(skb)->nr_frags) {
764 int i;
765
766 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
767 skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i];
768 get_page(skb_shinfo(n)->frags[i].page);
769 }
770 skb_shinfo(n)->nr_frags = i;
771 }
772
David S. Millerfbb398a2009-06-09 00:18:59 -0700773 if (skb_has_frags(skb)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700774 skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list;
775 skb_clone_fraglist(n);
776 }
777
778 copy_skb_header(n, skb);
779out:
780 return n;
781}
David S. Millerb4ac530fc2009-02-10 02:09:24 -0800782EXPORT_SYMBOL(pskb_copy);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700783
784/**
785 * pskb_expand_head - reallocate header of &sk_buff
786 * @skb: buffer to reallocate
787 * @nhead: room to add at head
788 * @ntail: room to add at tail
789 * @gfp_mask: allocation priority
790 *
791 * Expands (or creates identical copy, if &nhead and &ntail are zero)
792 * header of skb. &sk_buff itself is not changed. &sk_buff MUST have
793 * reference count of 1. Returns zero in the case of success or error,
794 * if expansion failed. In the last case, &sk_buff is not changed.
795 *
796 * All the pointers pointing into skb header may change and must be
797 * reloaded after call to this function.
798 */
799
Victor Fusco86a76ca2005-07-08 14:57:47 -0700800int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
Al Virodd0fc662005-10-07 07:46:04 +0100801 gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700802{
803 int i;
804 u8 *data;
Arnaldo Carvalho de Melo4305b542007-04-19 20:43:29 -0700805#ifdef NET_SKBUFF_DATA_USES_OFFSET
806 int size = nhead + skb->end + ntail;
807#else
Linus Torvalds1da177e2005-04-16 15:20:36 -0700808 int size = nhead + (skb->end - skb->head) + ntail;
Arnaldo Carvalho de Melo4305b542007-04-19 20:43:29 -0700809#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700810 long off;
811
Herbert Xu4edd87a2008-10-01 07:09:38 -0700812 BUG_ON(nhead < 0);
813
Linus Torvalds1da177e2005-04-16 15:20:36 -0700814 if (skb_shared(skb))
815 BUG();
816
817 size = SKB_DATA_ALIGN(size);
818
819 data = kmalloc(size + sizeof(struct skb_shared_info), gfp_mask);
820 if (!data)
821 goto nodata;
822
823 /* Copy only real data... and, alas, header. This should be
824 * optimized for the cases when header is void. */
Arnaldo Carvalho de Melo4305b542007-04-19 20:43:29 -0700825#ifdef NET_SKBUFF_DATA_USES_OFFSET
Mikael Petterssonb6ccc672007-05-19 13:55:25 -0700826 memcpy(data + nhead, skb->head, skb->tail);
Arnaldo Carvalho de Melo4305b542007-04-19 20:43:29 -0700827#else
Mikael Petterssonb6ccc672007-05-19 13:55:25 -0700828 memcpy(data + nhead, skb->head, skb->tail - skb->head);
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -0700829#endif
Arnaldo Carvalho de Melo4305b542007-04-19 20:43:29 -0700830 memcpy(data + size, skb_end_pointer(skb),
831 sizeof(struct skb_shared_info));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700832
833 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
834 get_page(skb_shinfo(skb)->frags[i].page);
835
David S. Millerfbb398a2009-06-09 00:18:59 -0700836 if (skb_has_frags(skb))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700837 skb_clone_fraglist(skb);
838
839 skb_release_data(skb);
840
841 off = (data + nhead) - skb->head;
842
843 skb->head = data;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700844 skb->data += off;
Arnaldo Carvalho de Melo4305b542007-04-19 20:43:29 -0700845#ifdef NET_SKBUFF_DATA_USES_OFFSET
846 skb->end = size;
Patrick McHardy56eb8882007-04-09 11:45:04 -0700847 off = nhead;
Arnaldo Carvalho de Melo4305b542007-04-19 20:43:29 -0700848#else
849 skb->end = skb->head + size;
Patrick McHardy56eb8882007-04-09 11:45:04 -0700850#endif
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -0700851 /* {transport,network,mac}_header and tail are relative to skb->head */
852 skb->tail += off;
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -0700853 skb->transport_header += off;
854 skb->network_header += off;
Stephen Hemminger603a8bb2009-06-17 12:17:34 +0000855 if (skb_mac_header_was_set(skb))
856 skb->mac_header += off;
Herbert Xu172a8632007-10-15 01:46:08 -0700857 skb->csum_start += nhead;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700858 skb->cloned = 0;
Patrick McHardy334a8132007-06-25 04:35:20 -0700859 skb->hdr_len = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700860 skb->nohdr = 0;
861 atomic_set(&skb_shinfo(skb)->dataref, 1);
862 return 0;
863
864nodata:
865 return -ENOMEM;
866}
David S. Millerb4ac530fc2009-02-10 02:09:24 -0800867EXPORT_SYMBOL(pskb_expand_head);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700868
869/* Make private copy of skb with writable head and some headroom */
870
871struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom)
872{
873 struct sk_buff *skb2;
874 int delta = headroom - skb_headroom(skb);
875
876 if (delta <= 0)
877 skb2 = pskb_copy(skb, GFP_ATOMIC);
878 else {
879 skb2 = skb_clone(skb, GFP_ATOMIC);
880 if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0,
881 GFP_ATOMIC)) {
882 kfree_skb(skb2);
883 skb2 = NULL;
884 }
885 }
886 return skb2;
887}
David S. Millerb4ac530fc2009-02-10 02:09:24 -0800888EXPORT_SYMBOL(skb_realloc_headroom);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700889
890/**
891 * skb_copy_expand - copy and expand sk_buff
892 * @skb: buffer to copy
893 * @newheadroom: new free bytes at head
894 * @newtailroom: new free bytes at tail
895 * @gfp_mask: allocation priority
896 *
897 * Make a copy of both an &sk_buff and its data and while doing so
898 * allocate additional space.
899 *
900 * This is used when the caller wishes to modify the data and needs a
901 * private copy of the data to alter as well as more space for new fields.
902 * Returns %NULL on failure or the pointer to the buffer
903 * on success. The returned buffer has a reference count of 1.
904 *
905 * You must pass %GFP_ATOMIC as the allocation priority if this function
906 * is called from an interrupt.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700907 */
908struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
Victor Fusco86a76ca2005-07-08 14:57:47 -0700909 int newheadroom, int newtailroom,
Al Virodd0fc662005-10-07 07:46:04 +0100910 gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700911{
912 /*
913 * Allocate the copy buffer
914 */
915 struct sk_buff *n = alloc_skb(newheadroom + skb->len + newtailroom,
916 gfp_mask);
Patrick McHardyefd1e8d2007-04-10 18:30:09 -0700917 int oldheadroom = skb_headroom(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700918 int head_copy_len, head_copy_off;
Herbert Xu52886052007-09-16 16:32:11 -0700919 int off;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700920
921 if (!n)
922 return NULL;
923
924 skb_reserve(n, newheadroom);
925
926 /* Set the tail pointer and length */
927 skb_put(n, skb->len);
928
Patrick McHardyefd1e8d2007-04-10 18:30:09 -0700929 head_copy_len = oldheadroom;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700930 head_copy_off = 0;
931 if (newheadroom <= head_copy_len)
932 head_copy_len = newheadroom;
933 else
934 head_copy_off = newheadroom - head_copy_len;
935
936 /* Copy the linear header and data. */
937 if (skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off,
938 skb->len + head_copy_len))
939 BUG();
940
941 copy_skb_header(n, skb);
942
Patrick McHardyefd1e8d2007-04-10 18:30:09 -0700943 off = newheadroom - oldheadroom;
Herbert Xu52886052007-09-16 16:32:11 -0700944 n->csum_start += off;
945#ifdef NET_SKBUFF_DATA_USES_OFFSET
Patrick McHardyefd1e8d2007-04-10 18:30:09 -0700946 n->transport_header += off;
947 n->network_header += off;
Stephen Hemminger603a8bb2009-06-17 12:17:34 +0000948 if (skb_mac_header_was_set(skb))
949 n->mac_header += off;
Herbert Xu52886052007-09-16 16:32:11 -0700950#endif
Patrick McHardyefd1e8d2007-04-10 18:30:09 -0700951
Linus Torvalds1da177e2005-04-16 15:20:36 -0700952 return n;
953}
David S. Millerb4ac530fc2009-02-10 02:09:24 -0800954EXPORT_SYMBOL(skb_copy_expand);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700955
956/**
957 * skb_pad - zero pad the tail of an skb
958 * @skb: buffer to pad
959 * @pad: space to pad
960 *
961 * Ensure that a buffer is followed by a padding area that is zero
962 * filled. Used by network drivers which may DMA or transfer data
963 * beyond the buffer end onto the wire.
964 *
Herbert Xu5b057c62006-06-23 02:06:41 -0700965 * May return error in out of memory cases. The skb is freed on error.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700966 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900967
Herbert Xu5b057c62006-06-23 02:06:41 -0700968int skb_pad(struct sk_buff *skb, int pad)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700969{
Herbert Xu5b057c62006-06-23 02:06:41 -0700970 int err;
971 int ntail;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900972
Linus Torvalds1da177e2005-04-16 15:20:36 -0700973 /* If the skbuff is non linear tailroom is always zero.. */
Herbert Xu5b057c62006-06-23 02:06:41 -0700974 if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700975 memset(skb->data+skb->len, 0, pad);
Herbert Xu5b057c62006-06-23 02:06:41 -0700976 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700977 }
Herbert Xu5b057c62006-06-23 02:06:41 -0700978
Arnaldo Carvalho de Melo4305b542007-04-19 20:43:29 -0700979 ntail = skb->data_len + pad - (skb->end - skb->tail);
Herbert Xu5b057c62006-06-23 02:06:41 -0700980 if (likely(skb_cloned(skb) || ntail > 0)) {
981 err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC);
982 if (unlikely(err))
983 goto free_skb;
984 }
985
986 /* FIXME: The use of this function with non-linear skb's really needs
987 * to be audited.
988 */
989 err = skb_linearize(skb);
990 if (unlikely(err))
991 goto free_skb;
992
993 memset(skb->data + skb->len, 0, pad);
994 return 0;
995
996free_skb:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700997 kfree_skb(skb);
Herbert Xu5b057c62006-06-23 02:06:41 -0700998 return err;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900999}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08001000EXPORT_SYMBOL(skb_pad);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001001
Ilpo Järvinen0dde3e12008-03-27 17:43:41 -07001002/**
1003 * skb_put - add data to a buffer
1004 * @skb: buffer to use
1005 * @len: amount of data to add
1006 *
1007 * This function extends the used data area of the buffer. If this would
1008 * exceed the total buffer size the kernel will panic. A pointer to the
1009 * first byte of the extra data is returned.
1010 */
1011unsigned char *skb_put(struct sk_buff *skb, unsigned int len)
1012{
1013 unsigned char *tmp = skb_tail_pointer(skb);
1014 SKB_LINEAR_ASSERT(skb);
1015 skb->tail += len;
1016 skb->len += len;
1017 if (unlikely(skb->tail > skb->end))
1018 skb_over_panic(skb, len, __builtin_return_address(0));
1019 return tmp;
1020}
1021EXPORT_SYMBOL(skb_put);
1022
Ilpo Järvinen6be8ac22008-03-27 17:47:24 -07001023/**
Ilpo Järvinenc2aa2702008-03-27 17:52:40 -07001024 * skb_push - add data to the start of a buffer
1025 * @skb: buffer to use
1026 * @len: amount of data to add
1027 *
1028 * This function extends the used data area of the buffer at the buffer
1029 * start. If this would exceed the total buffer headroom the kernel will
1030 * panic. A pointer to the first byte of the extra data is returned.
1031 */
1032unsigned char *skb_push(struct sk_buff *skb, unsigned int len)
1033{
1034 skb->data -= len;
1035 skb->len += len;
1036 if (unlikely(skb->data<skb->head))
1037 skb_under_panic(skb, len, __builtin_return_address(0));
1038 return skb->data;
1039}
1040EXPORT_SYMBOL(skb_push);
1041
1042/**
Ilpo Järvinen6be8ac22008-03-27 17:47:24 -07001043 * skb_pull - remove data from the start of a buffer
1044 * @skb: buffer to use
1045 * @len: amount of data to remove
1046 *
1047 * This function removes data from the start of a buffer, returning
1048 * the memory to the headroom. A pointer to the next data in the buffer
1049 * is returned. Once the data has been pulled future pushes will overwrite
1050 * the old data.
1051 */
1052unsigned char *skb_pull(struct sk_buff *skb, unsigned int len)
1053{
1054 return unlikely(len > skb->len) ? NULL : __skb_pull(skb, len);
1055}
1056EXPORT_SYMBOL(skb_pull);
1057
Ilpo Järvinen419ae742008-03-27 17:54:01 -07001058/**
1059 * skb_trim - remove end from a buffer
1060 * @skb: buffer to alter
1061 * @len: new length
1062 *
1063 * Cut the length of a buffer down by removing data from the tail. If
1064 * the buffer is already under the length specified it is not modified.
1065 * The skb must be linear.
1066 */
1067void skb_trim(struct sk_buff *skb, unsigned int len)
1068{
1069 if (skb->len > len)
1070 __skb_trim(skb, len);
1071}
1072EXPORT_SYMBOL(skb_trim);
1073
Herbert Xu3cc0e872006-06-09 16:13:38 -07001074/* Trims skb to length len. It can change skb pointers.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001075 */
1076
Herbert Xu3cc0e872006-06-09 16:13:38 -07001077int ___pskb_trim(struct sk_buff *skb, unsigned int len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001078{
Herbert Xu27b437c2006-07-13 19:26:39 -07001079 struct sk_buff **fragp;
1080 struct sk_buff *frag;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001081 int offset = skb_headlen(skb);
1082 int nfrags = skb_shinfo(skb)->nr_frags;
1083 int i;
Herbert Xu27b437c2006-07-13 19:26:39 -07001084 int err;
1085
1086 if (skb_cloned(skb) &&
1087 unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC))))
1088 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001089
Herbert Xuf4d26fb2006-07-30 20:20:28 -07001090 i = 0;
1091 if (offset >= len)
1092 goto drop_pages;
1093
1094 for (; i < nfrags; i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001095 int end = offset + skb_shinfo(skb)->frags[i].size;
Herbert Xu27b437c2006-07-13 19:26:39 -07001096
1097 if (end < len) {
1098 offset = end;
1099 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001100 }
Herbert Xu27b437c2006-07-13 19:26:39 -07001101
Herbert Xuf4d26fb2006-07-30 20:20:28 -07001102 skb_shinfo(skb)->frags[i++].size = len - offset;
Herbert Xu27b437c2006-07-13 19:26:39 -07001103
Herbert Xuf4d26fb2006-07-30 20:20:28 -07001104drop_pages:
Herbert Xu27b437c2006-07-13 19:26:39 -07001105 skb_shinfo(skb)->nr_frags = i;
1106
1107 for (; i < nfrags; i++)
1108 put_page(skb_shinfo(skb)->frags[i].page);
1109
David S. Millerfbb398a2009-06-09 00:18:59 -07001110 if (skb_has_frags(skb))
Herbert Xu27b437c2006-07-13 19:26:39 -07001111 skb_drop_fraglist(skb);
Herbert Xuf4d26fb2006-07-30 20:20:28 -07001112 goto done;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001113 }
1114
Herbert Xu27b437c2006-07-13 19:26:39 -07001115 for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp);
1116 fragp = &frag->next) {
1117 int end = offset + frag->len;
1118
1119 if (skb_shared(frag)) {
1120 struct sk_buff *nfrag;
1121
1122 nfrag = skb_clone(frag, GFP_ATOMIC);
1123 if (unlikely(!nfrag))
1124 return -ENOMEM;
1125
1126 nfrag->next = frag->next;
Herbert Xuf4d26fb2006-07-30 20:20:28 -07001127 kfree_skb(frag);
Herbert Xu27b437c2006-07-13 19:26:39 -07001128 frag = nfrag;
1129 *fragp = frag;
1130 }
1131
1132 if (end < len) {
1133 offset = end;
1134 continue;
1135 }
1136
1137 if (end > len &&
1138 unlikely((err = pskb_trim(frag, len - offset))))
1139 return err;
1140
1141 if (frag->next)
1142 skb_drop_list(&frag->next);
1143 break;
1144 }
1145
Herbert Xuf4d26fb2006-07-30 20:20:28 -07001146done:
Herbert Xu27b437c2006-07-13 19:26:39 -07001147 if (len > skb_headlen(skb)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001148 skb->data_len -= skb->len - len;
1149 skb->len = len;
1150 } else {
Herbert Xu27b437c2006-07-13 19:26:39 -07001151 skb->len = len;
1152 skb->data_len = 0;
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -07001153 skb_set_tail_pointer(skb, len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001154 }
1155
1156 return 0;
1157}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08001158EXPORT_SYMBOL(___pskb_trim);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001159
1160/**
1161 * __pskb_pull_tail - advance tail of skb header
1162 * @skb: buffer to reallocate
1163 * @delta: number of bytes to advance tail
1164 *
1165 * The function makes a sense only on a fragmented &sk_buff,
1166 * it expands header moving its tail forward and copying necessary
1167 * data from fragmented part.
1168 *
1169 * &sk_buff MUST have reference count of 1.
1170 *
1171 * Returns %NULL (and &sk_buff does not change) if pull failed
1172 * or value of new tail of skb in the case of success.
1173 *
1174 * All the pointers pointing into skb header may change and must be
1175 * reloaded after call to this function.
1176 */
1177
1178/* Moves tail of skb head forward, copying data from fragmented part,
1179 * when it is necessary.
1180 * 1. It may fail due to malloc failure.
1181 * 2. It may change skb pointers.
1182 *
1183 * It is pretty complicated. Luckily, it is called only in exceptional cases.
1184 */
1185unsigned char *__pskb_pull_tail(struct sk_buff *skb, int delta)
1186{
1187 /* If skb has not enough free space at tail, get new one
1188 * plus 128 bytes for future expansions. If we have enough
1189 * room at tail, reallocate without expansion only if skb is cloned.
1190 */
Arnaldo Carvalho de Melo4305b542007-04-19 20:43:29 -07001191 int i, k, eat = (skb->tail + delta) - skb->end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001192
1193 if (eat > 0 || skb_cloned(skb)) {
1194 if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0,
1195 GFP_ATOMIC))
1196 return NULL;
1197 }
1198
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -07001199 if (skb_copy_bits(skb, skb_headlen(skb), skb_tail_pointer(skb), delta))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001200 BUG();
1201
1202 /* Optimization: no fragments, no reasons to preestimate
1203 * size of pulled pages. Superb.
1204 */
David S. Millerfbb398a2009-06-09 00:18:59 -07001205 if (!skb_has_frags(skb))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001206 goto pull_pages;
1207
1208 /* Estimate size of pulled pages. */
1209 eat = delta;
1210 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1211 if (skb_shinfo(skb)->frags[i].size >= eat)
1212 goto pull_pages;
1213 eat -= skb_shinfo(skb)->frags[i].size;
1214 }
1215
1216 /* If we need update frag list, we are in troubles.
1217 * Certainly, it possible to add an offset to skb data,
1218 * but taking into account that pulling is expected to
1219 * be very rare operation, it is worth to fight against
1220 * further bloating skb head and crucify ourselves here instead.
1221 * Pure masohism, indeed. 8)8)
1222 */
1223 if (eat) {
1224 struct sk_buff *list = skb_shinfo(skb)->frag_list;
1225 struct sk_buff *clone = NULL;
1226 struct sk_buff *insp = NULL;
1227
1228 do {
Kris Katterjohn09a62662006-01-08 22:24:28 -08001229 BUG_ON(!list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001230
1231 if (list->len <= eat) {
1232 /* Eaten as whole. */
1233 eat -= list->len;
1234 list = list->next;
1235 insp = list;
1236 } else {
1237 /* Eaten partially. */
1238
1239 if (skb_shared(list)) {
1240 /* Sucks! We need to fork list. :-( */
1241 clone = skb_clone(list, GFP_ATOMIC);
1242 if (!clone)
1243 return NULL;
1244 insp = list->next;
1245 list = clone;
1246 } else {
1247 /* This may be pulled without
1248 * problems. */
1249 insp = list;
1250 }
1251 if (!pskb_pull(list, eat)) {
Wei Yongjunf3fbbe02009-02-25 00:37:32 +00001252 kfree_skb(clone);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001253 return NULL;
1254 }
1255 break;
1256 }
1257 } while (eat);
1258
1259 /* Free pulled out fragments. */
1260 while ((list = skb_shinfo(skb)->frag_list) != insp) {
1261 skb_shinfo(skb)->frag_list = list->next;
1262 kfree_skb(list);
1263 }
1264 /* And insert new clone at head. */
1265 if (clone) {
1266 clone->next = list;
1267 skb_shinfo(skb)->frag_list = clone;
1268 }
1269 }
1270 /* Success! Now we may commit changes to skb data. */
1271
1272pull_pages:
1273 eat = delta;
1274 k = 0;
1275 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1276 if (skb_shinfo(skb)->frags[i].size <= eat) {
1277 put_page(skb_shinfo(skb)->frags[i].page);
1278 eat -= skb_shinfo(skb)->frags[i].size;
1279 } else {
1280 skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i];
1281 if (eat) {
1282 skb_shinfo(skb)->frags[k].page_offset += eat;
1283 skb_shinfo(skb)->frags[k].size -= eat;
1284 eat = 0;
1285 }
1286 k++;
1287 }
1288 }
1289 skb_shinfo(skb)->nr_frags = k;
1290
1291 skb->tail += delta;
1292 skb->data_len -= delta;
1293
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -07001294 return skb_tail_pointer(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001295}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08001296EXPORT_SYMBOL(__pskb_pull_tail);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001297
1298/* Copy some data bits from skb to kernel buffer. */
1299
1300int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len)
1301{
David S. Miller1a028e52007-04-27 15:21:23 -07001302 int start = skb_headlen(skb);
David S. Millerfbb398a2009-06-09 00:18:59 -07001303 struct sk_buff *frag_iter;
1304 int i, copy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001305
1306 if (offset > (int)skb->len - len)
1307 goto fault;
1308
1309 /* Copy header. */
David S. Miller1a028e52007-04-27 15:21:23 -07001310 if ((copy = start - offset) > 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001311 if (copy > len)
1312 copy = len;
Arnaldo Carvalho de Melod626f622007-03-27 18:55:52 -03001313 skb_copy_from_linear_data_offset(skb, offset, to, copy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001314 if ((len -= copy) == 0)
1315 return 0;
1316 offset += copy;
1317 to += copy;
1318 }
1319
1320 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
David S. Miller1a028e52007-04-27 15:21:23 -07001321 int end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001322
Ilpo Järvinen547b7922008-07-25 21:43:18 -07001323 WARN_ON(start > offset + len);
David S. Miller1a028e52007-04-27 15:21:23 -07001324
1325 end = start + skb_shinfo(skb)->frags[i].size;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001326 if ((copy = end - offset) > 0) {
1327 u8 *vaddr;
1328
1329 if (copy > len)
1330 copy = len;
1331
1332 vaddr = kmap_skb_frag(&skb_shinfo(skb)->frags[i]);
1333 memcpy(to,
David S. Miller1a028e52007-04-27 15:21:23 -07001334 vaddr + skb_shinfo(skb)->frags[i].page_offset+
1335 offset - start, copy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001336 kunmap_skb_frag(vaddr);
1337
1338 if ((len -= copy) == 0)
1339 return 0;
1340 offset += copy;
1341 to += copy;
1342 }
David S. Miller1a028e52007-04-27 15:21:23 -07001343 start = end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001344 }
1345
David S. Millerfbb398a2009-06-09 00:18:59 -07001346 skb_walk_frags(skb, frag_iter) {
1347 int end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001348
David S. Millerfbb398a2009-06-09 00:18:59 -07001349 WARN_ON(start > offset + len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001350
David S. Millerfbb398a2009-06-09 00:18:59 -07001351 end = start + frag_iter->len;
1352 if ((copy = end - offset) > 0) {
1353 if (copy > len)
1354 copy = len;
1355 if (skb_copy_bits(frag_iter, offset - start, to, copy))
1356 goto fault;
1357 if ((len -= copy) == 0)
1358 return 0;
1359 offset += copy;
1360 to += copy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001361 }
David S. Millerfbb398a2009-06-09 00:18:59 -07001362 start = end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001363 }
1364 if (!len)
1365 return 0;
1366
1367fault:
1368 return -EFAULT;
1369}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08001370EXPORT_SYMBOL(skb_copy_bits);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001371
Jens Axboe9c55e012007-11-06 23:30:13 -08001372/*
1373 * Callback from splice_to_pipe(), if we need to release some pages
1374 * at the end of the spd in case we error'ed out in filling the pipe.
1375 */
1376static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i)
1377{
Jarek Poplawski8b9d3722009-01-19 17:03:56 -08001378 put_page(spd->pages[i]);
1379}
Jens Axboe9c55e012007-11-06 23:30:13 -08001380
Jarek Poplawski4fb66992009-02-01 00:41:42 -08001381static inline struct page *linear_to_page(struct page *page, unsigned int *len,
1382 unsigned int *offset,
Jarek Poplawski7a67e562009-04-30 05:41:19 -07001383 struct sk_buff *skb, struct sock *sk)
Jarek Poplawski8b9d3722009-01-19 17:03:56 -08001384{
Jarek Poplawski4fb66992009-02-01 00:41:42 -08001385 struct page *p = sk->sk_sndmsg_page;
1386 unsigned int off;
Jarek Poplawski8b9d3722009-01-19 17:03:56 -08001387
Jarek Poplawski4fb66992009-02-01 00:41:42 -08001388 if (!p) {
1389new_page:
1390 p = sk->sk_sndmsg_page = alloc_pages(sk->sk_allocation, 0);
1391 if (!p)
1392 return NULL;
1393
1394 off = sk->sk_sndmsg_off = 0;
1395 /* hold one ref to this page until it's full */
1396 } else {
1397 unsigned int mlen;
1398
1399 off = sk->sk_sndmsg_off;
1400 mlen = PAGE_SIZE - off;
1401 if (mlen < 64 && mlen < *len) {
1402 put_page(p);
1403 goto new_page;
1404 }
1405
1406 *len = min_t(unsigned int, *len, mlen);
1407 }
1408
1409 memcpy(page_address(p) + off, page_address(page) + *offset, *len);
1410 sk->sk_sndmsg_off += *len;
1411 *offset = off;
1412 get_page(p);
Jarek Poplawski8b9d3722009-01-19 17:03:56 -08001413
1414 return p;
Jens Axboe9c55e012007-11-06 23:30:13 -08001415}
1416
1417/*
1418 * Fill page/offset/length into spd, if it can hold more pages.
1419 */
1420static inline int spd_fill_page(struct splice_pipe_desc *spd, struct page *page,
Jarek Poplawski4fb66992009-02-01 00:41:42 -08001421 unsigned int *len, unsigned int offset,
Jarek Poplawski7a67e562009-04-30 05:41:19 -07001422 struct sk_buff *skb, int linear,
1423 struct sock *sk)
Jens Axboe9c55e012007-11-06 23:30:13 -08001424{
1425 if (unlikely(spd->nr_pages == PIPE_BUFFERS))
1426 return 1;
1427
Jarek Poplawski8b9d3722009-01-19 17:03:56 -08001428 if (linear) {
Jarek Poplawski7a67e562009-04-30 05:41:19 -07001429 page = linear_to_page(page, len, &offset, skb, sk);
Jarek Poplawski8b9d3722009-01-19 17:03:56 -08001430 if (!page)
1431 return 1;
1432 } else
1433 get_page(page);
1434
Jens Axboe9c55e012007-11-06 23:30:13 -08001435 spd->pages[spd->nr_pages] = page;
Jarek Poplawski4fb66992009-02-01 00:41:42 -08001436 spd->partial[spd->nr_pages].len = *len;
Jens Axboe9c55e012007-11-06 23:30:13 -08001437 spd->partial[spd->nr_pages].offset = offset;
Jens Axboe9c55e012007-11-06 23:30:13 -08001438 spd->nr_pages++;
Jarek Poplawski8b9d3722009-01-19 17:03:56 -08001439
Jens Axboe9c55e012007-11-06 23:30:13 -08001440 return 0;
1441}
1442
Octavian Purdila2870c432008-07-15 00:49:11 -07001443static inline void __segment_seek(struct page **page, unsigned int *poff,
1444 unsigned int *plen, unsigned int off)
Jens Axboe9c55e012007-11-06 23:30:13 -08001445{
Jarek Poplawskice3dd392009-02-12 16:51:43 -08001446 unsigned long n;
1447
Octavian Purdila2870c432008-07-15 00:49:11 -07001448 *poff += off;
Jarek Poplawskice3dd392009-02-12 16:51:43 -08001449 n = *poff / PAGE_SIZE;
1450 if (n)
1451 *page = nth_page(*page, n);
1452
Octavian Purdila2870c432008-07-15 00:49:11 -07001453 *poff = *poff % PAGE_SIZE;
1454 *plen -= off;
1455}
Jens Axboe9c55e012007-11-06 23:30:13 -08001456
Octavian Purdila2870c432008-07-15 00:49:11 -07001457static inline int __splice_segment(struct page *page, unsigned int poff,
1458 unsigned int plen, unsigned int *off,
1459 unsigned int *len, struct sk_buff *skb,
Jarek Poplawski7a67e562009-04-30 05:41:19 -07001460 struct splice_pipe_desc *spd, int linear,
1461 struct sock *sk)
Octavian Purdila2870c432008-07-15 00:49:11 -07001462{
1463 if (!*len)
1464 return 1;
1465
1466 /* skip this segment if already processed */
1467 if (*off >= plen) {
1468 *off -= plen;
1469 return 0;
Octavian Purdiladb43a282008-06-27 17:27:21 -07001470 }
Jens Axboe9c55e012007-11-06 23:30:13 -08001471
Octavian Purdila2870c432008-07-15 00:49:11 -07001472 /* ignore any bits we already processed */
1473 if (*off) {
1474 __segment_seek(&page, &poff, &plen, *off);
1475 *off = 0;
1476 }
1477
1478 do {
1479 unsigned int flen = min(*len, plen);
1480
1481 /* the linear region may spread across several pages */
1482 flen = min_t(unsigned int, flen, PAGE_SIZE - poff);
1483
Jarek Poplawski7a67e562009-04-30 05:41:19 -07001484 if (spd_fill_page(spd, page, &flen, poff, skb, linear, sk))
Octavian Purdila2870c432008-07-15 00:49:11 -07001485 return 1;
1486
1487 __segment_seek(&page, &poff, &plen, flen);
1488 *len -= flen;
1489
1490 } while (*len && plen);
1491
1492 return 0;
1493}
1494
1495/*
1496 * Map linear and fragment data from the skb to spd. It reports failure if the
1497 * pipe is full or if we already spliced the requested length.
1498 */
Harvey Harrison7b1c65f2008-07-16 20:12:30 -07001499static int __skb_splice_bits(struct sk_buff *skb, unsigned int *offset,
Jarek Poplawski7a67e562009-04-30 05:41:19 -07001500 unsigned int *len, struct splice_pipe_desc *spd,
1501 struct sock *sk)
Octavian Purdila2870c432008-07-15 00:49:11 -07001502{
1503 int seg;
1504
Jens Axboe9c55e012007-11-06 23:30:13 -08001505 /*
Octavian Purdila2870c432008-07-15 00:49:11 -07001506 * map the linear part
Jens Axboe9c55e012007-11-06 23:30:13 -08001507 */
Octavian Purdila2870c432008-07-15 00:49:11 -07001508 if (__splice_segment(virt_to_page(skb->data),
1509 (unsigned long) skb->data & (PAGE_SIZE - 1),
1510 skb_headlen(skb),
Jarek Poplawski7a67e562009-04-30 05:41:19 -07001511 offset, len, skb, spd, 1, sk))
Octavian Purdila2870c432008-07-15 00:49:11 -07001512 return 1;
Jens Axboe9c55e012007-11-06 23:30:13 -08001513
1514 /*
1515 * then map the fragments
1516 */
Jens Axboe9c55e012007-11-06 23:30:13 -08001517 for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) {
1518 const skb_frag_t *f = &skb_shinfo(skb)->frags[seg];
1519
Octavian Purdila2870c432008-07-15 00:49:11 -07001520 if (__splice_segment(f->page, f->page_offset, f->size,
Jarek Poplawski7a67e562009-04-30 05:41:19 -07001521 offset, len, skb, spd, 0, sk))
Octavian Purdila2870c432008-07-15 00:49:11 -07001522 return 1;
Jens Axboe9c55e012007-11-06 23:30:13 -08001523 }
1524
Octavian Purdila2870c432008-07-15 00:49:11 -07001525 return 0;
Jens Axboe9c55e012007-11-06 23:30:13 -08001526}
1527
1528/*
1529 * Map data from the skb to a pipe. Should handle both the linear part,
1530 * the fragments, and the frag list. It does NOT handle frag lists within
1531 * the frag list, if such a thing exists. We'd probably need to recurse to
1532 * handle that cleanly.
1533 */
Jarek Poplawski8b9d3722009-01-19 17:03:56 -08001534int skb_splice_bits(struct sk_buff *skb, unsigned int offset,
Jens Axboe9c55e012007-11-06 23:30:13 -08001535 struct pipe_inode_info *pipe, unsigned int tlen,
1536 unsigned int flags)
1537{
1538 struct partial_page partial[PIPE_BUFFERS];
1539 struct page *pages[PIPE_BUFFERS];
1540 struct splice_pipe_desc spd = {
1541 .pages = pages,
1542 .partial = partial,
1543 .flags = flags,
1544 .ops = &sock_pipe_buf_ops,
1545 .spd_release = sock_spd_release,
1546 };
David S. Millerfbb398a2009-06-09 00:18:59 -07001547 struct sk_buff *frag_iter;
Jarek Poplawski7a67e562009-04-30 05:41:19 -07001548 struct sock *sk = skb->sk;
Jens Axboe9c55e012007-11-06 23:30:13 -08001549
1550 /*
1551 * __skb_splice_bits() only fails if the output has no room left,
1552 * so no point in going over the frag_list for the error case.
1553 */
Jarek Poplawski7a67e562009-04-30 05:41:19 -07001554 if (__skb_splice_bits(skb, &offset, &tlen, &spd, sk))
Jens Axboe9c55e012007-11-06 23:30:13 -08001555 goto done;
1556 else if (!tlen)
1557 goto done;
1558
1559 /*
1560 * now see if we have a frag_list to map
1561 */
David S. Millerfbb398a2009-06-09 00:18:59 -07001562 skb_walk_frags(skb, frag_iter) {
1563 if (!tlen)
1564 break;
1565 if (__skb_splice_bits(frag_iter, &offset, &tlen, &spd, sk))
1566 break;
Jens Axboe9c55e012007-11-06 23:30:13 -08001567 }
1568
1569done:
Jens Axboe9c55e012007-11-06 23:30:13 -08001570 if (spd.nr_pages) {
1571 int ret;
1572
1573 /*
1574 * Drop the socket lock, otherwise we have reverse
1575 * locking dependencies between sk_lock and i_mutex
1576 * here as compared to sendfile(). We enter here
1577 * with the socket lock held, and splice_to_pipe() will
1578 * grab the pipe inode lock. For sendfile() emulation,
1579 * we call into ->sendpage() with the i_mutex lock held
1580 * and networking will grab the socket lock.
1581 */
Octavian Purdila293ad602008-06-04 15:45:58 -07001582 release_sock(sk);
Jens Axboe9c55e012007-11-06 23:30:13 -08001583 ret = splice_to_pipe(pipe, &spd);
Octavian Purdila293ad602008-06-04 15:45:58 -07001584 lock_sock(sk);
Jens Axboe9c55e012007-11-06 23:30:13 -08001585 return ret;
1586 }
1587
1588 return 0;
1589}
1590
Herbert Xu357b40a2005-04-19 22:30:14 -07001591/**
1592 * skb_store_bits - store bits from kernel buffer to skb
1593 * @skb: destination buffer
1594 * @offset: offset in destination
1595 * @from: source buffer
1596 * @len: number of bytes to copy
1597 *
1598 * Copy the specified number of bytes from the source buffer to the
1599 * destination skb. This function handles all the messy bits of
1600 * traversing fragment lists and such.
1601 */
1602
Stephen Hemminger0c6fcc82007-04-20 16:40:01 -07001603int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len)
Herbert Xu357b40a2005-04-19 22:30:14 -07001604{
David S. Miller1a028e52007-04-27 15:21:23 -07001605 int start = skb_headlen(skb);
David S. Millerfbb398a2009-06-09 00:18:59 -07001606 struct sk_buff *frag_iter;
1607 int i, copy;
Herbert Xu357b40a2005-04-19 22:30:14 -07001608
1609 if (offset > (int)skb->len - len)
1610 goto fault;
1611
David S. Miller1a028e52007-04-27 15:21:23 -07001612 if ((copy = start - offset) > 0) {
Herbert Xu357b40a2005-04-19 22:30:14 -07001613 if (copy > len)
1614 copy = len;
Arnaldo Carvalho de Melo27d7ff42007-03-31 11:55:19 -03001615 skb_copy_to_linear_data_offset(skb, offset, from, copy);
Herbert Xu357b40a2005-04-19 22:30:14 -07001616 if ((len -= copy) == 0)
1617 return 0;
1618 offset += copy;
1619 from += copy;
1620 }
1621
1622 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1623 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
David S. Miller1a028e52007-04-27 15:21:23 -07001624 int end;
Herbert Xu357b40a2005-04-19 22:30:14 -07001625
Ilpo Järvinen547b7922008-07-25 21:43:18 -07001626 WARN_ON(start > offset + len);
David S. Miller1a028e52007-04-27 15:21:23 -07001627
1628 end = start + frag->size;
Herbert Xu357b40a2005-04-19 22:30:14 -07001629 if ((copy = end - offset) > 0) {
1630 u8 *vaddr;
1631
1632 if (copy > len)
1633 copy = len;
1634
1635 vaddr = kmap_skb_frag(frag);
David S. Miller1a028e52007-04-27 15:21:23 -07001636 memcpy(vaddr + frag->page_offset + offset - start,
1637 from, copy);
Herbert Xu357b40a2005-04-19 22:30:14 -07001638 kunmap_skb_frag(vaddr);
1639
1640 if ((len -= copy) == 0)
1641 return 0;
1642 offset += copy;
1643 from += copy;
1644 }
David S. Miller1a028e52007-04-27 15:21:23 -07001645 start = end;
Herbert Xu357b40a2005-04-19 22:30:14 -07001646 }
1647
David S. Millerfbb398a2009-06-09 00:18:59 -07001648 skb_walk_frags(skb, frag_iter) {
1649 int end;
Herbert Xu357b40a2005-04-19 22:30:14 -07001650
David S. Millerfbb398a2009-06-09 00:18:59 -07001651 WARN_ON(start > offset + len);
Herbert Xu357b40a2005-04-19 22:30:14 -07001652
David S. Millerfbb398a2009-06-09 00:18:59 -07001653 end = start + frag_iter->len;
1654 if ((copy = end - offset) > 0) {
1655 if (copy > len)
1656 copy = len;
1657 if (skb_store_bits(frag_iter, offset - start,
1658 from, copy))
1659 goto fault;
1660 if ((len -= copy) == 0)
1661 return 0;
1662 offset += copy;
1663 from += copy;
Herbert Xu357b40a2005-04-19 22:30:14 -07001664 }
David S. Millerfbb398a2009-06-09 00:18:59 -07001665 start = end;
Herbert Xu357b40a2005-04-19 22:30:14 -07001666 }
1667 if (!len)
1668 return 0;
1669
1670fault:
1671 return -EFAULT;
1672}
Herbert Xu357b40a2005-04-19 22:30:14 -07001673EXPORT_SYMBOL(skb_store_bits);
1674
Linus Torvalds1da177e2005-04-16 15:20:36 -07001675/* Checksum skb data. */
1676
Al Viro2bbbc862006-11-14 21:37:14 -08001677__wsum skb_checksum(const struct sk_buff *skb, int offset,
1678 int len, __wsum csum)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001679{
David S. Miller1a028e52007-04-27 15:21:23 -07001680 int start = skb_headlen(skb);
1681 int i, copy = start - offset;
David S. Millerfbb398a2009-06-09 00:18:59 -07001682 struct sk_buff *frag_iter;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001683 int pos = 0;
1684
1685 /* Checksum header. */
1686 if (copy > 0) {
1687 if (copy > len)
1688 copy = len;
1689 csum = csum_partial(skb->data + offset, copy, csum);
1690 if ((len -= copy) == 0)
1691 return csum;
1692 offset += copy;
1693 pos = copy;
1694 }
1695
1696 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
David S. Miller1a028e52007-04-27 15:21:23 -07001697 int end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001698
Ilpo Järvinen547b7922008-07-25 21:43:18 -07001699 WARN_ON(start > offset + len);
David S. Miller1a028e52007-04-27 15:21:23 -07001700
1701 end = start + skb_shinfo(skb)->frags[i].size;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001702 if ((copy = end - offset) > 0) {
Al Viro44bb9362006-11-14 21:36:14 -08001703 __wsum csum2;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001704 u8 *vaddr;
1705 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1706
1707 if (copy > len)
1708 copy = len;
1709 vaddr = kmap_skb_frag(frag);
David S. Miller1a028e52007-04-27 15:21:23 -07001710 csum2 = csum_partial(vaddr + frag->page_offset +
1711 offset - start, copy, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001712 kunmap_skb_frag(vaddr);
1713 csum = csum_block_add(csum, csum2, pos);
1714 if (!(len -= copy))
1715 return csum;
1716 offset += copy;
1717 pos += copy;
1718 }
David S. Miller1a028e52007-04-27 15:21:23 -07001719 start = end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001720 }
1721
David S. Millerfbb398a2009-06-09 00:18:59 -07001722 skb_walk_frags(skb, frag_iter) {
1723 int end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001724
David S. Millerfbb398a2009-06-09 00:18:59 -07001725 WARN_ON(start > offset + len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001726
David S. Millerfbb398a2009-06-09 00:18:59 -07001727 end = start + frag_iter->len;
1728 if ((copy = end - offset) > 0) {
1729 __wsum csum2;
1730 if (copy > len)
1731 copy = len;
1732 csum2 = skb_checksum(frag_iter, offset - start,
1733 copy, 0);
1734 csum = csum_block_add(csum, csum2, pos);
1735 if ((len -= copy) == 0)
1736 return csum;
1737 offset += copy;
1738 pos += copy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001739 }
David S. Millerfbb398a2009-06-09 00:18:59 -07001740 start = end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001741 }
Kris Katterjohn09a62662006-01-08 22:24:28 -08001742 BUG_ON(len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001743
1744 return csum;
1745}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08001746EXPORT_SYMBOL(skb_checksum);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001747
1748/* Both of above in one bottle. */
1749
Al Viro81d77662006-11-14 21:37:33 -08001750__wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset,
1751 u8 *to, int len, __wsum csum)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001752{
David S. Miller1a028e52007-04-27 15:21:23 -07001753 int start = skb_headlen(skb);
1754 int i, copy = start - offset;
David S. Millerfbb398a2009-06-09 00:18:59 -07001755 struct sk_buff *frag_iter;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001756 int pos = 0;
1757
1758 /* Copy header. */
1759 if (copy > 0) {
1760 if (copy > len)
1761 copy = len;
1762 csum = csum_partial_copy_nocheck(skb->data + offset, to,
1763 copy, csum);
1764 if ((len -= copy) == 0)
1765 return csum;
1766 offset += copy;
1767 to += copy;
1768 pos = copy;
1769 }
1770
1771 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
David S. Miller1a028e52007-04-27 15:21:23 -07001772 int end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001773
Ilpo Järvinen547b7922008-07-25 21:43:18 -07001774 WARN_ON(start > offset + len);
David S. Miller1a028e52007-04-27 15:21:23 -07001775
1776 end = start + skb_shinfo(skb)->frags[i].size;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001777 if ((copy = end - offset) > 0) {
Al Viro50842052006-11-14 21:36:34 -08001778 __wsum csum2;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001779 u8 *vaddr;
1780 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1781
1782 if (copy > len)
1783 copy = len;
1784 vaddr = kmap_skb_frag(frag);
1785 csum2 = csum_partial_copy_nocheck(vaddr +
David S. Miller1a028e52007-04-27 15:21:23 -07001786 frag->page_offset +
1787 offset - start, to,
1788 copy, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001789 kunmap_skb_frag(vaddr);
1790 csum = csum_block_add(csum, csum2, pos);
1791 if (!(len -= copy))
1792 return csum;
1793 offset += copy;
1794 to += copy;
1795 pos += copy;
1796 }
David S. Miller1a028e52007-04-27 15:21:23 -07001797 start = end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001798 }
1799
David S. Millerfbb398a2009-06-09 00:18:59 -07001800 skb_walk_frags(skb, frag_iter) {
1801 __wsum csum2;
1802 int end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001803
David S. Millerfbb398a2009-06-09 00:18:59 -07001804 WARN_ON(start > offset + len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001805
David S. Millerfbb398a2009-06-09 00:18:59 -07001806 end = start + frag_iter->len;
1807 if ((copy = end - offset) > 0) {
1808 if (copy > len)
1809 copy = len;
1810 csum2 = skb_copy_and_csum_bits(frag_iter,
1811 offset - start,
1812 to, copy, 0);
1813 csum = csum_block_add(csum, csum2, pos);
1814 if ((len -= copy) == 0)
1815 return csum;
1816 offset += copy;
1817 to += copy;
1818 pos += copy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001819 }
David S. Millerfbb398a2009-06-09 00:18:59 -07001820 start = end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001821 }
Kris Katterjohn09a62662006-01-08 22:24:28 -08001822 BUG_ON(len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001823 return csum;
1824}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08001825EXPORT_SYMBOL(skb_copy_and_csum_bits);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001826
1827void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to)
1828{
Al Virod3bc23e2006-11-14 21:24:49 -08001829 __wsum csum;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001830 long csstart;
1831
Patrick McHardy84fa7932006-08-29 16:44:56 -07001832 if (skb->ip_summed == CHECKSUM_PARTIAL)
Herbert Xu663ead32007-04-09 11:59:07 -07001833 csstart = skb->csum_start - skb_headroom(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001834 else
1835 csstart = skb_headlen(skb);
1836
Kris Katterjohn09a62662006-01-08 22:24:28 -08001837 BUG_ON(csstart > skb_headlen(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001838
Arnaldo Carvalho de Melod626f622007-03-27 18:55:52 -03001839 skb_copy_from_linear_data(skb, to, csstart);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001840
1841 csum = 0;
1842 if (csstart != skb->len)
1843 csum = skb_copy_and_csum_bits(skb, csstart, to + csstart,
1844 skb->len - csstart, 0);
1845
Patrick McHardy84fa7932006-08-29 16:44:56 -07001846 if (skb->ip_summed == CHECKSUM_PARTIAL) {
Al Viroff1dcad2006-11-20 18:07:29 -08001847 long csstuff = csstart + skb->csum_offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001848
Al Virod3bc23e2006-11-14 21:24:49 -08001849 *((__sum16 *)(to + csstuff)) = csum_fold(csum);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001850 }
1851}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08001852EXPORT_SYMBOL(skb_copy_and_csum_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001853
1854/**
1855 * skb_dequeue - remove from the head of the queue
1856 * @list: list to dequeue from
1857 *
1858 * Remove the head of the list. The list lock is taken so the function
1859 * may be used safely with other locking list functions. The head item is
1860 * returned or %NULL if the list is empty.
1861 */
1862
1863struct sk_buff *skb_dequeue(struct sk_buff_head *list)
1864{
1865 unsigned long flags;
1866 struct sk_buff *result;
1867
1868 spin_lock_irqsave(&list->lock, flags);
1869 result = __skb_dequeue(list);
1870 spin_unlock_irqrestore(&list->lock, flags);
1871 return result;
1872}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08001873EXPORT_SYMBOL(skb_dequeue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001874
1875/**
1876 * skb_dequeue_tail - remove from the tail of the queue
1877 * @list: list to dequeue from
1878 *
1879 * Remove the tail of the list. The list lock is taken so the function
1880 * may be used safely with other locking list functions. The tail item is
1881 * returned or %NULL if the list is empty.
1882 */
1883struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list)
1884{
1885 unsigned long flags;
1886 struct sk_buff *result;
1887
1888 spin_lock_irqsave(&list->lock, flags);
1889 result = __skb_dequeue_tail(list);
1890 spin_unlock_irqrestore(&list->lock, flags);
1891 return result;
1892}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08001893EXPORT_SYMBOL(skb_dequeue_tail);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001894
1895/**
1896 * skb_queue_purge - empty a list
1897 * @list: list to empty
1898 *
1899 * Delete all buffers on an &sk_buff list. Each buffer is removed from
1900 * the list and one reference dropped. This function takes the list
1901 * lock and is atomic with respect to other list locking functions.
1902 */
1903void skb_queue_purge(struct sk_buff_head *list)
1904{
1905 struct sk_buff *skb;
1906 while ((skb = skb_dequeue(list)) != NULL)
1907 kfree_skb(skb);
1908}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08001909EXPORT_SYMBOL(skb_queue_purge);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001910
1911/**
1912 * skb_queue_head - queue a buffer at the list head
1913 * @list: list to use
1914 * @newsk: buffer to queue
1915 *
1916 * Queue a buffer at the start of the list. This function takes the
1917 * list lock and can be used safely with other locking &sk_buff functions
1918 * safely.
1919 *
1920 * A buffer cannot be placed on two lists at the same time.
1921 */
1922void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk)
1923{
1924 unsigned long flags;
1925
1926 spin_lock_irqsave(&list->lock, flags);
1927 __skb_queue_head(list, newsk);
1928 spin_unlock_irqrestore(&list->lock, flags);
1929}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08001930EXPORT_SYMBOL(skb_queue_head);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001931
1932/**
1933 * skb_queue_tail - queue a buffer at the list tail
1934 * @list: list to use
1935 * @newsk: buffer to queue
1936 *
1937 * Queue a buffer at the tail of the list. This function takes the
1938 * list lock and can be used safely with other locking &sk_buff functions
1939 * safely.
1940 *
1941 * A buffer cannot be placed on two lists at the same time.
1942 */
1943void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk)
1944{
1945 unsigned long flags;
1946
1947 spin_lock_irqsave(&list->lock, flags);
1948 __skb_queue_tail(list, newsk);
1949 spin_unlock_irqrestore(&list->lock, flags);
1950}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08001951EXPORT_SYMBOL(skb_queue_tail);
David S. Miller8728b832005-08-09 19:25:21 -07001952
Linus Torvalds1da177e2005-04-16 15:20:36 -07001953/**
1954 * skb_unlink - remove a buffer from a list
1955 * @skb: buffer to remove
David S. Miller8728b832005-08-09 19:25:21 -07001956 * @list: list to use
Linus Torvalds1da177e2005-04-16 15:20:36 -07001957 *
David S. Miller8728b832005-08-09 19:25:21 -07001958 * Remove a packet from a list. The list locks are taken and this
1959 * function is atomic with respect to other list locked calls
Linus Torvalds1da177e2005-04-16 15:20:36 -07001960 *
David S. Miller8728b832005-08-09 19:25:21 -07001961 * You must know what list the SKB is on.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001962 */
David S. Miller8728b832005-08-09 19:25:21 -07001963void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001964{
David S. Miller8728b832005-08-09 19:25:21 -07001965 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001966
David S. Miller8728b832005-08-09 19:25:21 -07001967 spin_lock_irqsave(&list->lock, flags);
1968 __skb_unlink(skb, list);
1969 spin_unlock_irqrestore(&list->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001970}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08001971EXPORT_SYMBOL(skb_unlink);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001972
Linus Torvalds1da177e2005-04-16 15:20:36 -07001973/**
1974 * skb_append - append a buffer
1975 * @old: buffer to insert after
1976 * @newsk: buffer to insert
David S. Miller8728b832005-08-09 19:25:21 -07001977 * @list: list to use
Linus Torvalds1da177e2005-04-16 15:20:36 -07001978 *
1979 * Place a packet after a given packet in a list. The list locks are taken
1980 * and this function is atomic with respect to other list locked calls.
1981 * A buffer cannot be placed on two lists at the same time.
1982 */
David S. Miller8728b832005-08-09 19:25:21 -07001983void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001984{
1985 unsigned long flags;
1986
David S. Miller8728b832005-08-09 19:25:21 -07001987 spin_lock_irqsave(&list->lock, flags);
Gerrit Renker7de6c032008-04-14 00:05:09 -07001988 __skb_queue_after(list, old, newsk);
David S. Miller8728b832005-08-09 19:25:21 -07001989 spin_unlock_irqrestore(&list->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001990}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08001991EXPORT_SYMBOL(skb_append);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001992
1993/**
1994 * skb_insert - insert a buffer
1995 * @old: buffer to insert before
1996 * @newsk: buffer to insert
David S. Miller8728b832005-08-09 19:25:21 -07001997 * @list: list to use
Linus Torvalds1da177e2005-04-16 15:20:36 -07001998 *
David S. Miller8728b832005-08-09 19:25:21 -07001999 * Place a packet before a given packet in a list. The list locks are
2000 * taken and this function is atomic with respect to other list locked
2001 * calls.
2002 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07002003 * A buffer cannot be placed on two lists at the same time.
2004 */
David S. Miller8728b832005-08-09 19:25:21 -07002005void skb_insert(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002006{
2007 unsigned long flags;
2008
David S. Miller8728b832005-08-09 19:25:21 -07002009 spin_lock_irqsave(&list->lock, flags);
2010 __skb_insert(newsk, old->prev, old, list);
2011 spin_unlock_irqrestore(&list->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002012}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08002013EXPORT_SYMBOL(skb_insert);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002014
Linus Torvalds1da177e2005-04-16 15:20:36 -07002015static inline void skb_split_inside_header(struct sk_buff *skb,
2016 struct sk_buff* skb1,
2017 const u32 len, const int pos)
2018{
2019 int i;
2020
Arnaldo Carvalho de Melod626f622007-03-27 18:55:52 -03002021 skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len),
2022 pos - len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002023 /* And move data appendix as is. */
2024 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
2025 skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i];
2026
2027 skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags;
2028 skb_shinfo(skb)->nr_frags = 0;
2029 skb1->data_len = skb->data_len;
2030 skb1->len += skb1->data_len;
2031 skb->data_len = 0;
2032 skb->len = len;
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -07002033 skb_set_tail_pointer(skb, len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002034}
2035
2036static inline void skb_split_no_header(struct sk_buff *skb,
2037 struct sk_buff* skb1,
2038 const u32 len, int pos)
2039{
2040 int i, k = 0;
2041 const int nfrags = skb_shinfo(skb)->nr_frags;
2042
2043 skb_shinfo(skb)->nr_frags = 0;
2044 skb1->len = skb1->data_len = skb->len - len;
2045 skb->len = len;
2046 skb->data_len = len - pos;
2047
2048 for (i = 0; i < nfrags; i++) {
2049 int size = skb_shinfo(skb)->frags[i].size;
2050
2051 if (pos + size > len) {
2052 skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i];
2053
2054 if (pos < len) {
2055 /* Split frag.
2056 * We have two variants in this case:
2057 * 1. Move all the frag to the second
2058 * part, if it is possible. F.e.
2059 * this approach is mandatory for TUX,
2060 * where splitting is expensive.
2061 * 2. Split is accurately. We make this.
2062 */
2063 get_page(skb_shinfo(skb)->frags[i].page);
2064 skb_shinfo(skb1)->frags[0].page_offset += len - pos;
2065 skb_shinfo(skb1)->frags[0].size -= len - pos;
2066 skb_shinfo(skb)->frags[i].size = len - pos;
2067 skb_shinfo(skb)->nr_frags++;
2068 }
2069 k++;
2070 } else
2071 skb_shinfo(skb)->nr_frags++;
2072 pos += size;
2073 }
2074 skb_shinfo(skb1)->nr_frags = k;
2075}
2076
2077/**
2078 * skb_split - Split fragmented skb to two parts at length len.
2079 * @skb: the buffer to split
2080 * @skb1: the buffer to receive the second part
2081 * @len: new length for skb
2082 */
2083void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len)
2084{
2085 int pos = skb_headlen(skb);
2086
2087 if (len < pos) /* Split line is inside header. */
2088 skb_split_inside_header(skb, skb1, len, pos);
2089 else /* Second chunk has no header, nothing to copy. */
2090 skb_split_no_header(skb, skb1, len, pos);
2091}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08002092EXPORT_SYMBOL(skb_split);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002093
Ilpo Järvinen9f782db2008-11-25 13:57:01 -08002094/* Shifting from/to a cloned skb is a no-go.
2095 *
2096 * Caller cannot keep skb_shinfo related pointers past calling here!
2097 */
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08002098static int skb_prepare_for_shift(struct sk_buff *skb)
2099{
Ilpo Järvinen0ace2852008-11-24 21:30:21 -08002100 return skb_cloned(skb) && pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08002101}
2102
2103/**
2104 * skb_shift - Shifts paged data partially from skb to another
2105 * @tgt: buffer into which tail data gets added
2106 * @skb: buffer from which the paged data comes from
2107 * @shiftlen: shift up to this many bytes
2108 *
2109 * Attempts to shift up to shiftlen worth of bytes, which may be less than
2110 * the length of the skb, from tgt to skb. Returns number bytes shifted.
2111 * It's up to caller to free skb if everything was shifted.
2112 *
2113 * If @tgt runs out of frags, the whole operation is aborted.
2114 *
2115 * Skb cannot include anything else but paged data while tgt is allowed
2116 * to have non-paged data as well.
2117 *
2118 * TODO: full sized shift could be optimized but that would need
2119 * specialized skb free'er to handle frags without up-to-date nr_frags.
2120 */
2121int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen)
2122{
2123 int from, to, merge, todo;
2124 struct skb_frag_struct *fragfrom, *fragto;
2125
2126 BUG_ON(shiftlen > skb->len);
2127 BUG_ON(skb_headlen(skb)); /* Would corrupt stream */
2128
2129 todo = shiftlen;
2130 from = 0;
2131 to = skb_shinfo(tgt)->nr_frags;
2132 fragfrom = &skb_shinfo(skb)->frags[from];
2133
2134 /* Actual merge is delayed until the point when we know we can
2135 * commit all, so that we don't have to undo partial changes
2136 */
2137 if (!to ||
2138 !skb_can_coalesce(tgt, to, fragfrom->page, fragfrom->page_offset)) {
2139 merge = -1;
2140 } else {
2141 merge = to - 1;
2142
2143 todo -= fragfrom->size;
2144 if (todo < 0) {
2145 if (skb_prepare_for_shift(skb) ||
2146 skb_prepare_for_shift(tgt))
2147 return 0;
2148
Ilpo Järvinen9f782db2008-11-25 13:57:01 -08002149 /* All previous frag pointers might be stale! */
2150 fragfrom = &skb_shinfo(skb)->frags[from];
Ilpo Järvinen832d11c2008-11-24 21:20:15 -08002151 fragto = &skb_shinfo(tgt)->frags[merge];
2152
2153 fragto->size += shiftlen;
2154 fragfrom->size -= shiftlen;
2155 fragfrom->page_offset += shiftlen;
2156
2157 goto onlymerged;
2158 }
2159
2160 from++;
2161 }
2162
2163 /* Skip full, not-fitting skb to avoid expensive operations */
2164 if ((shiftlen == skb->len) &&
2165 (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to))
2166 return 0;
2167
2168 if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt))
2169 return 0;
2170
2171 while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) {
2172 if (to == MAX_SKB_FRAGS)
2173 return 0;
2174
2175 fragfrom = &skb_shinfo(skb)->frags[from];
2176 fragto = &skb_shinfo(tgt)->frags[to];
2177
2178 if (todo >= fragfrom->size) {
2179 *fragto = *fragfrom;
2180 todo -= fragfrom->size;
2181 from++;
2182 to++;
2183
2184 } else {
2185 get_page(fragfrom->page);
2186 fragto->page = fragfrom->page;
2187 fragto->page_offset = fragfrom->page_offset;
2188 fragto->size = todo;
2189
2190 fragfrom->page_offset += todo;
2191 fragfrom->size -= todo;
2192 todo = 0;
2193
2194 to++;
2195 break;
2196 }
2197 }
2198
2199 /* Ready to "commit" this state change to tgt */
2200 skb_shinfo(tgt)->nr_frags = to;
2201
2202 if (merge >= 0) {
2203 fragfrom = &skb_shinfo(skb)->frags[0];
2204 fragto = &skb_shinfo(tgt)->frags[merge];
2205
2206 fragto->size += fragfrom->size;
2207 put_page(fragfrom->page);
2208 }
2209
2210 /* Reposition in the original skb */
2211 to = 0;
2212 while (from < skb_shinfo(skb)->nr_frags)
2213 skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++];
2214 skb_shinfo(skb)->nr_frags = to;
2215
2216 BUG_ON(todo > 0 && !skb_shinfo(skb)->nr_frags);
2217
2218onlymerged:
2219 /* Most likely the tgt won't ever need its checksum anymore, skb on
2220 * the other hand might need it if it needs to be resent
2221 */
2222 tgt->ip_summed = CHECKSUM_PARTIAL;
2223 skb->ip_summed = CHECKSUM_PARTIAL;
2224
2225 /* Yak, is it really working this way? Some helper please? */
2226 skb->len -= shiftlen;
2227 skb->data_len -= shiftlen;
2228 skb->truesize -= shiftlen;
2229 tgt->len += shiftlen;
2230 tgt->data_len += shiftlen;
2231 tgt->truesize += shiftlen;
2232
2233 return shiftlen;
2234}
2235
Thomas Graf677e90e2005-06-23 20:59:51 -07002236/**
2237 * skb_prepare_seq_read - Prepare a sequential read of skb data
2238 * @skb: the buffer to read
2239 * @from: lower offset of data to be read
2240 * @to: upper offset of data to be read
2241 * @st: state variable
2242 *
2243 * Initializes the specified state variable. Must be called before
2244 * invoking skb_seq_read() for the first time.
2245 */
2246void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
2247 unsigned int to, struct skb_seq_state *st)
2248{
2249 st->lower_offset = from;
2250 st->upper_offset = to;
2251 st->root_skb = st->cur_skb = skb;
2252 st->frag_idx = st->stepped_offset = 0;
2253 st->frag_data = NULL;
2254}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08002255EXPORT_SYMBOL(skb_prepare_seq_read);
Thomas Graf677e90e2005-06-23 20:59:51 -07002256
2257/**
2258 * skb_seq_read - Sequentially read skb data
2259 * @consumed: number of bytes consumed by the caller so far
2260 * @data: destination pointer for data to be returned
2261 * @st: state variable
2262 *
2263 * Reads a block of skb data at &consumed relative to the
2264 * lower offset specified to skb_prepare_seq_read(). Assigns
2265 * the head of the data block to &data and returns the length
2266 * of the block or 0 if the end of the skb data or the upper
2267 * offset has been reached.
2268 *
2269 * The caller is not required to consume all of the data
2270 * returned, i.e. &consumed is typically set to the number
2271 * of bytes already consumed and the next call to
2272 * skb_seq_read() will return the remaining part of the block.
2273 *
Randy Dunlapbc2cda12008-02-13 15:03:25 -08002274 * Note 1: The size of each block of data returned can be arbitary,
Thomas Graf677e90e2005-06-23 20:59:51 -07002275 * this limitation is the cost for zerocopy seqeuental
2276 * reads of potentially non linear data.
2277 *
Randy Dunlapbc2cda12008-02-13 15:03:25 -08002278 * Note 2: Fragment lists within fragments are not implemented
Thomas Graf677e90e2005-06-23 20:59:51 -07002279 * at the moment, state->root_skb could be replaced with
2280 * a stack for this purpose.
2281 */
2282unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
2283 struct skb_seq_state *st)
2284{
2285 unsigned int block_limit, abs_offset = consumed + st->lower_offset;
2286 skb_frag_t *frag;
2287
2288 if (unlikely(abs_offset >= st->upper_offset))
2289 return 0;
2290
2291next_skb:
Herbert Xu95e3b242009-01-29 16:07:52 -08002292 block_limit = skb_headlen(st->cur_skb) + st->stepped_offset;
Thomas Graf677e90e2005-06-23 20:59:51 -07002293
Thomas Chenault995b3372009-05-18 21:43:27 -07002294 if (abs_offset < block_limit && !st->frag_data) {
Herbert Xu95e3b242009-01-29 16:07:52 -08002295 *data = st->cur_skb->data + (abs_offset - st->stepped_offset);
Thomas Graf677e90e2005-06-23 20:59:51 -07002296 return block_limit - abs_offset;
2297 }
2298
2299 if (st->frag_idx == 0 && !st->frag_data)
2300 st->stepped_offset += skb_headlen(st->cur_skb);
2301
2302 while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) {
2303 frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx];
2304 block_limit = frag->size + st->stepped_offset;
2305
2306 if (abs_offset < block_limit) {
2307 if (!st->frag_data)
2308 st->frag_data = kmap_skb_frag(frag);
2309
2310 *data = (u8 *) st->frag_data + frag->page_offset +
2311 (abs_offset - st->stepped_offset);
2312
2313 return block_limit - abs_offset;
2314 }
2315
2316 if (st->frag_data) {
2317 kunmap_skb_frag(st->frag_data);
2318 st->frag_data = NULL;
2319 }
2320
2321 st->frag_idx++;
2322 st->stepped_offset += frag->size;
2323 }
2324
Olaf Kirch5b5a60d2007-06-23 23:11:52 -07002325 if (st->frag_data) {
2326 kunmap_skb_frag(st->frag_data);
2327 st->frag_data = NULL;
2328 }
2329
David S. Millerfbb398a2009-06-09 00:18:59 -07002330 if (st->root_skb == st->cur_skb && skb_has_frags(st->root_skb)) {
Shyam Iyer71b33462009-01-29 16:12:42 -08002331 st->cur_skb = skb_shinfo(st->root_skb)->frag_list;
Thomas Graf677e90e2005-06-23 20:59:51 -07002332 st->frag_idx = 0;
2333 goto next_skb;
Shyam Iyer71b33462009-01-29 16:12:42 -08002334 } else if (st->cur_skb->next) {
2335 st->cur_skb = st->cur_skb->next;
Herbert Xu95e3b242009-01-29 16:07:52 -08002336 st->frag_idx = 0;
Thomas Graf677e90e2005-06-23 20:59:51 -07002337 goto next_skb;
2338 }
2339
2340 return 0;
2341}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08002342EXPORT_SYMBOL(skb_seq_read);
Thomas Graf677e90e2005-06-23 20:59:51 -07002343
2344/**
2345 * skb_abort_seq_read - Abort a sequential read of skb data
2346 * @st: state variable
2347 *
2348 * Must be called if skb_seq_read() was not called until it
2349 * returned 0.
2350 */
2351void skb_abort_seq_read(struct skb_seq_state *st)
2352{
2353 if (st->frag_data)
2354 kunmap_skb_frag(st->frag_data);
2355}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08002356EXPORT_SYMBOL(skb_abort_seq_read);
Thomas Graf677e90e2005-06-23 20:59:51 -07002357
Thomas Graf3fc7e8a2005-06-23 21:00:17 -07002358#define TS_SKB_CB(state) ((struct skb_seq_state *) &((state)->cb))
2359
2360static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text,
2361 struct ts_config *conf,
2362 struct ts_state *state)
2363{
2364 return skb_seq_read(offset, text, TS_SKB_CB(state));
2365}
2366
2367static void skb_ts_finish(struct ts_config *conf, struct ts_state *state)
2368{
2369 skb_abort_seq_read(TS_SKB_CB(state));
2370}
2371
2372/**
2373 * skb_find_text - Find a text pattern in skb data
2374 * @skb: the buffer to look in
2375 * @from: search offset
2376 * @to: search limit
2377 * @config: textsearch configuration
2378 * @state: uninitialized textsearch state variable
2379 *
2380 * Finds a pattern in the skb data according to the specified
2381 * textsearch configuration. Use textsearch_next() to retrieve
2382 * subsequent occurrences of the pattern. Returns the offset
2383 * to the first occurrence or UINT_MAX if no match was found.
2384 */
2385unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
2386 unsigned int to, struct ts_config *config,
2387 struct ts_state *state)
2388{
Phil Oesterf72b9482006-06-26 00:00:57 -07002389 unsigned int ret;
2390
Thomas Graf3fc7e8a2005-06-23 21:00:17 -07002391 config->get_next_block = skb_ts_get_next_block;
2392 config->finish = skb_ts_finish;
2393
2394 skb_prepare_seq_read(skb, from, to, TS_SKB_CB(state));
2395
Phil Oesterf72b9482006-06-26 00:00:57 -07002396 ret = textsearch_find(config, state);
2397 return (ret <= to - from ? ret : UINT_MAX);
Thomas Graf3fc7e8a2005-06-23 21:00:17 -07002398}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08002399EXPORT_SYMBOL(skb_find_text);
Thomas Graf3fc7e8a2005-06-23 21:00:17 -07002400
Ananda Rajue89e9cf2005-10-18 15:46:41 -07002401/**
2402 * skb_append_datato_frags: - append the user data to a skb
2403 * @sk: sock structure
2404 * @skb: skb structure to be appened with user data.
2405 * @getfrag: call back function to be used for getting the user data
2406 * @from: pointer to user message iov
2407 * @length: length of the iov message
2408 *
2409 * Description: This procedure append the user data in the fragment part
2410 * of the skb if any page alloc fails user this procedure returns -ENOMEM
2411 */
2412int skb_append_datato_frags(struct sock *sk, struct sk_buff *skb,
Martin Waitzdab96302005-12-05 13:40:12 -08002413 int (*getfrag)(void *from, char *to, int offset,
Ananda Rajue89e9cf2005-10-18 15:46:41 -07002414 int len, int odd, struct sk_buff *skb),
2415 void *from, int length)
2416{
2417 int frg_cnt = 0;
2418 skb_frag_t *frag = NULL;
2419 struct page *page = NULL;
2420 int copy, left;
2421 int offset = 0;
2422 int ret;
2423
2424 do {
2425 /* Return error if we don't have space for new frag */
2426 frg_cnt = skb_shinfo(skb)->nr_frags;
2427 if (frg_cnt >= MAX_SKB_FRAGS)
2428 return -EFAULT;
2429
2430 /* allocate a new page for next frag */
2431 page = alloc_pages(sk->sk_allocation, 0);
2432
2433 /* If alloc_page fails just return failure and caller will
2434 * free previous allocated pages by doing kfree_skb()
2435 */
2436 if (page == NULL)
2437 return -ENOMEM;
2438
2439 /* initialize the next frag */
2440 sk->sk_sndmsg_page = page;
2441 sk->sk_sndmsg_off = 0;
2442 skb_fill_page_desc(skb, frg_cnt, page, 0, 0);
2443 skb->truesize += PAGE_SIZE;
2444 atomic_add(PAGE_SIZE, &sk->sk_wmem_alloc);
2445
2446 /* get the new initialized frag */
2447 frg_cnt = skb_shinfo(skb)->nr_frags;
2448 frag = &skb_shinfo(skb)->frags[frg_cnt - 1];
2449
2450 /* copy the user data to page */
2451 left = PAGE_SIZE - frag->page_offset;
2452 copy = (length > left)? left : length;
2453
2454 ret = getfrag(from, (page_address(frag->page) +
2455 frag->page_offset + frag->size),
2456 offset, copy, 0, skb);
2457 if (ret < 0)
2458 return -EFAULT;
2459
2460 /* copy was successful so update the size parameters */
2461 sk->sk_sndmsg_off += copy;
2462 frag->size += copy;
2463 skb->len += copy;
2464 skb->data_len += copy;
2465 offset += copy;
2466 length -= copy;
2467
2468 } while (length > 0);
2469
2470 return 0;
2471}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08002472EXPORT_SYMBOL(skb_append_datato_frags);
Ananda Rajue89e9cf2005-10-18 15:46:41 -07002473
Herbert Xucbb042f2006-03-20 22:43:56 -08002474/**
2475 * skb_pull_rcsum - pull skb and update receive checksum
2476 * @skb: buffer to update
Herbert Xucbb042f2006-03-20 22:43:56 -08002477 * @len: length of data pulled
2478 *
2479 * This function performs an skb_pull on the packet and updates
Urs Thuermannfee54fa2008-02-12 22:03:25 -08002480 * the CHECKSUM_COMPLETE checksum. It should be used on
Patrick McHardy84fa7932006-08-29 16:44:56 -07002481 * receive path processing instead of skb_pull unless you know
2482 * that the checksum difference is zero (e.g., a valid IP header)
2483 * or you are setting ip_summed to CHECKSUM_NONE.
Herbert Xucbb042f2006-03-20 22:43:56 -08002484 */
2485unsigned char *skb_pull_rcsum(struct sk_buff *skb, unsigned int len)
2486{
2487 BUG_ON(len > skb->len);
2488 skb->len -= len;
2489 BUG_ON(skb->len < skb->data_len);
2490 skb_postpull_rcsum(skb, skb->data, len);
2491 return skb->data += len;
2492}
2493
Arnaldo Carvalho de Melof94691a2006-03-20 22:47:55 -08002494EXPORT_SYMBOL_GPL(skb_pull_rcsum);
2495
Herbert Xuf4c50d92006-06-22 03:02:40 -07002496/**
2497 * skb_segment - Perform protocol segmentation on skb.
2498 * @skb: buffer to segment
Herbert Xu576a30e2006-06-27 13:22:38 -07002499 * @features: features for the output path (see dev->features)
Herbert Xuf4c50d92006-06-22 03:02:40 -07002500 *
2501 * This function performs segmentation on the given skb. It returns
Ben Hutchings4c821d72008-04-13 21:52:48 -07002502 * a pointer to the first in a list of new skbs for the segments.
2503 * In case of error it returns ERR_PTR(err).
Herbert Xuf4c50d92006-06-22 03:02:40 -07002504 */
Herbert Xu576a30e2006-06-27 13:22:38 -07002505struct sk_buff *skb_segment(struct sk_buff *skb, int features)
Herbert Xuf4c50d92006-06-22 03:02:40 -07002506{
2507 struct sk_buff *segs = NULL;
2508 struct sk_buff *tail = NULL;
Herbert Xu89319d382008-12-15 23:26:06 -08002509 struct sk_buff *fskb = skb_shinfo(skb)->frag_list;
Herbert Xuf4c50d92006-06-22 03:02:40 -07002510 unsigned int mss = skb_shinfo(skb)->gso_size;
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -07002511 unsigned int doffset = skb->data - skb_mac_header(skb);
Herbert Xuf4c50d92006-06-22 03:02:40 -07002512 unsigned int offset = doffset;
2513 unsigned int headroom;
2514 unsigned int len;
Herbert Xu576a30e2006-06-27 13:22:38 -07002515 int sg = features & NETIF_F_SG;
Herbert Xuf4c50d92006-06-22 03:02:40 -07002516 int nfrags = skb_shinfo(skb)->nr_frags;
2517 int err = -ENOMEM;
2518 int i = 0;
2519 int pos;
2520
2521 __skb_push(skb, doffset);
2522 headroom = skb_headroom(skb);
2523 pos = skb_headlen(skb);
2524
2525 do {
2526 struct sk_buff *nskb;
2527 skb_frag_t *frag;
Herbert Xuc8884ed2006-10-29 15:59:41 -08002528 int hsize;
Herbert Xuf4c50d92006-06-22 03:02:40 -07002529 int size;
2530
2531 len = skb->len - offset;
2532 if (len > mss)
2533 len = mss;
2534
2535 hsize = skb_headlen(skb) - offset;
2536 if (hsize < 0)
2537 hsize = 0;
Herbert Xuc8884ed2006-10-29 15:59:41 -08002538 if (hsize > len || !sg)
2539 hsize = len;
Herbert Xuf4c50d92006-06-22 03:02:40 -07002540
Herbert Xu89319d382008-12-15 23:26:06 -08002541 if (!hsize && i >= nfrags) {
2542 BUG_ON(fskb->len != len);
2543
2544 pos += len;
2545 nskb = skb_clone(fskb, GFP_ATOMIC);
2546 fskb = fskb->next;
2547
2548 if (unlikely(!nskb))
2549 goto err;
2550
2551 hsize = skb_end_pointer(nskb) - nskb->head;
2552 if (skb_cow_head(nskb, doffset + headroom)) {
2553 kfree_skb(nskb);
2554 goto err;
2555 }
2556
2557 nskb->truesize += skb_end_pointer(nskb) - nskb->head -
2558 hsize;
2559 skb_release_head_state(nskb);
2560 __skb_push(nskb, doffset);
2561 } else {
2562 nskb = alloc_skb(hsize + doffset + headroom,
2563 GFP_ATOMIC);
2564
2565 if (unlikely(!nskb))
2566 goto err;
2567
2568 skb_reserve(nskb, headroom);
2569 __skb_put(nskb, doffset);
2570 }
Herbert Xuf4c50d92006-06-22 03:02:40 -07002571
2572 if (segs)
2573 tail->next = nskb;
2574 else
2575 segs = nskb;
2576 tail = nskb;
2577
Herbert Xu6f85a122008-08-15 14:55:02 -07002578 __copy_skb_header(nskb, skb);
Herbert Xuf4c50d92006-06-22 03:02:40 -07002579 nskb->mac_len = skb->mac_len;
2580
Arnaldo Carvalho de Melo459a98e2007-03-19 15:30:44 -07002581 skb_reset_mac_header(nskb);
Arnaldo Carvalho de Meloddc7b8e2007-03-15 21:42:27 -03002582 skb_set_network_header(nskb, skb->mac_len);
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -07002583 nskb->transport_header = (nskb->network_header +
2584 skb_network_header_len(skb));
Herbert Xu89319d382008-12-15 23:26:06 -08002585 skb_copy_from_linear_data(skb, nskb->data, doffset);
2586
Herbert Xu2f181852009-03-28 23:39:18 -07002587 if (fskb != skb_shinfo(skb)->frag_list)
Herbert Xu89319d382008-12-15 23:26:06 -08002588 continue;
2589
Herbert Xuf4c50d92006-06-22 03:02:40 -07002590 if (!sg) {
Herbert Xu6f85a122008-08-15 14:55:02 -07002591 nskb->ip_summed = CHECKSUM_NONE;
Herbert Xuf4c50d92006-06-22 03:02:40 -07002592 nskb->csum = skb_copy_and_csum_bits(skb, offset,
2593 skb_put(nskb, len),
2594 len, 0);
2595 continue;
2596 }
2597
2598 frag = skb_shinfo(nskb)->frags;
Herbert Xuf4c50d92006-06-22 03:02:40 -07002599
Arnaldo Carvalho de Melod626f622007-03-27 18:55:52 -03002600 skb_copy_from_linear_data_offset(skb, offset,
2601 skb_put(nskb, hsize), hsize);
Herbert Xuf4c50d92006-06-22 03:02:40 -07002602
Herbert Xu89319d382008-12-15 23:26:06 -08002603 while (pos < offset + len && i < nfrags) {
Herbert Xuf4c50d92006-06-22 03:02:40 -07002604 *frag = skb_shinfo(skb)->frags[i];
2605 get_page(frag->page);
2606 size = frag->size;
2607
2608 if (pos < offset) {
2609 frag->page_offset += offset - pos;
2610 frag->size -= offset - pos;
2611 }
2612
Herbert Xu89319d382008-12-15 23:26:06 -08002613 skb_shinfo(nskb)->nr_frags++;
Herbert Xuf4c50d92006-06-22 03:02:40 -07002614
2615 if (pos + size <= offset + len) {
2616 i++;
2617 pos += size;
2618 } else {
2619 frag->size -= pos + size - (offset + len);
Herbert Xu89319d382008-12-15 23:26:06 -08002620 goto skip_fraglist;
Herbert Xuf4c50d92006-06-22 03:02:40 -07002621 }
2622
2623 frag++;
2624 }
2625
Herbert Xu89319d382008-12-15 23:26:06 -08002626 if (pos < offset + len) {
2627 struct sk_buff *fskb2 = fskb;
2628
2629 BUG_ON(pos + fskb->len != offset + len);
2630
2631 pos += fskb->len;
2632 fskb = fskb->next;
2633
2634 if (fskb2->next) {
2635 fskb2 = skb_clone(fskb2, GFP_ATOMIC);
2636 if (!fskb2)
2637 goto err;
2638 } else
2639 skb_get(fskb2);
2640
David S. Millerfbb398a2009-06-09 00:18:59 -07002641 SKB_FRAG_ASSERT(nskb);
Herbert Xu89319d382008-12-15 23:26:06 -08002642 skb_shinfo(nskb)->frag_list = fskb2;
2643 }
2644
2645skip_fraglist:
Herbert Xuf4c50d92006-06-22 03:02:40 -07002646 nskb->data_len = len - hsize;
2647 nskb->len += nskb->data_len;
2648 nskb->truesize += nskb->data_len;
2649 } while ((offset += len) < skb->len);
2650
2651 return segs;
2652
2653err:
2654 while ((skb = segs)) {
2655 segs = skb->next;
Patrick McHardyb08d5842007-02-27 09:57:37 -08002656 kfree_skb(skb);
Herbert Xuf4c50d92006-06-22 03:02:40 -07002657 }
2658 return ERR_PTR(err);
2659}
Herbert Xuf4c50d92006-06-22 03:02:40 -07002660EXPORT_SYMBOL_GPL(skb_segment);
2661
Herbert Xu71d93b32008-12-15 23:42:33 -08002662int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb)
2663{
2664 struct sk_buff *p = *head;
2665 struct sk_buff *nskb;
Herbert Xu9aaa1562009-05-26 18:50:33 +00002666 struct skb_shared_info *skbinfo = skb_shinfo(skb);
2667 struct skb_shared_info *pinfo = skb_shinfo(p);
Herbert Xu71d93b32008-12-15 23:42:33 -08002668 unsigned int headroom;
Herbert Xu86911732009-01-29 14:19:50 +00002669 unsigned int len = skb_gro_len(skb);
Herbert Xu67147ba2009-05-26 18:50:22 +00002670 unsigned int offset = skb_gro_offset(skb);
2671 unsigned int headlen = skb_headlen(skb);
Herbert Xu71d93b32008-12-15 23:42:33 -08002672
Herbert Xu86911732009-01-29 14:19:50 +00002673 if (p->len + len >= 65536)
Herbert Xu71d93b32008-12-15 23:42:33 -08002674 return -E2BIG;
2675
Herbert Xu9aaa1562009-05-26 18:50:33 +00002676 if (pinfo->frag_list)
Herbert Xu71d93b32008-12-15 23:42:33 -08002677 goto merge;
Herbert Xu67147ba2009-05-26 18:50:22 +00002678 else if (headlen <= offset) {
Herbert Xu42da6992009-05-26 18:50:19 +00002679 skb_frag_t *frag;
Herbert Xu66e92fc2009-05-26 18:50:32 +00002680 skb_frag_t *frag2;
Herbert Xu9aaa1562009-05-26 18:50:33 +00002681 int i = skbinfo->nr_frags;
2682 int nr_frags = pinfo->nr_frags + i;
Herbert Xu42da6992009-05-26 18:50:19 +00002683
Herbert Xu66e92fc2009-05-26 18:50:32 +00002684 offset -= headlen;
2685
2686 if (nr_frags > MAX_SKB_FRAGS)
Herbert Xu81705ad2009-01-29 14:19:51 +00002687 return -E2BIG;
2688
Herbert Xu9aaa1562009-05-26 18:50:33 +00002689 pinfo->nr_frags = nr_frags;
2690 skbinfo->nr_frags = 0;
Herbert Xuf5572062009-01-14 20:40:03 -08002691
Herbert Xu9aaa1562009-05-26 18:50:33 +00002692 frag = pinfo->frags + nr_frags;
2693 frag2 = skbinfo->frags + i;
Herbert Xu66e92fc2009-05-26 18:50:32 +00002694 do {
2695 *--frag = *--frag2;
2696 } while (--i);
2697
2698 frag->page_offset += offset;
2699 frag->size -= offset;
2700
Herbert Xuf5572062009-01-14 20:40:03 -08002701 skb->truesize -= skb->data_len;
2702 skb->len -= skb->data_len;
2703 skb->data_len = 0;
2704
Herbert Xu5d38a072009-01-04 16:13:40 -08002705 NAPI_GRO_CB(skb)->free = 1;
2706 goto done;
Herbert Xu69c0cab2009-11-17 05:18:18 -08002707 } else if (skb_gro_len(p) != pinfo->gso_size)
2708 return -E2BIG;
Herbert Xu71d93b32008-12-15 23:42:33 -08002709
2710 headroom = skb_headroom(p);
Herbert Xu86911732009-01-29 14:19:50 +00002711 nskb = netdev_alloc_skb(p->dev, headroom + skb_gro_offset(p));
Herbert Xu71d93b32008-12-15 23:42:33 -08002712 if (unlikely(!nskb))
2713 return -ENOMEM;
2714
2715 __copy_skb_header(nskb, p);
2716 nskb->mac_len = p->mac_len;
2717
2718 skb_reserve(nskb, headroom);
Herbert Xu86911732009-01-29 14:19:50 +00002719 __skb_put(nskb, skb_gro_offset(p));
Herbert Xu71d93b32008-12-15 23:42:33 -08002720
Herbert Xu86911732009-01-29 14:19:50 +00002721 skb_set_mac_header(nskb, skb_mac_header(p) - p->data);
Herbert Xu71d93b32008-12-15 23:42:33 -08002722 skb_set_network_header(nskb, skb_network_offset(p));
2723 skb_set_transport_header(nskb, skb_transport_offset(p));
2724
Herbert Xu86911732009-01-29 14:19:50 +00002725 __skb_pull(p, skb_gro_offset(p));
2726 memcpy(skb_mac_header(nskb), skb_mac_header(p),
2727 p->data - skb_mac_header(p));
Herbert Xu71d93b32008-12-15 23:42:33 -08002728
2729 *NAPI_GRO_CB(nskb) = *NAPI_GRO_CB(p);
2730 skb_shinfo(nskb)->frag_list = p;
Herbert Xu9aaa1562009-05-26 18:50:33 +00002731 skb_shinfo(nskb)->gso_size = pinfo->gso_size;
Herbert Xu71d93b32008-12-15 23:42:33 -08002732 skb_header_release(p);
2733 nskb->prev = p;
2734
2735 nskb->data_len += p->len;
2736 nskb->truesize += p->len;
2737 nskb->len += p->len;
2738
2739 *head = nskb;
2740 nskb->next = p->next;
2741 p->next = NULL;
2742
2743 p = nskb;
2744
2745merge:
Herbert Xu67147ba2009-05-26 18:50:22 +00002746 if (offset > headlen) {
Herbert Xu9aaa1562009-05-26 18:50:33 +00002747 skbinfo->frags[0].page_offset += offset - headlen;
2748 skbinfo->frags[0].size -= offset - headlen;
Herbert Xu67147ba2009-05-26 18:50:22 +00002749 offset = headlen;
Herbert Xu56035022009-02-05 21:26:52 -08002750 }
2751
Herbert Xu67147ba2009-05-26 18:50:22 +00002752 __skb_pull(skb, offset);
Herbert Xu56035022009-02-05 21:26:52 -08002753
Herbert Xu71d93b32008-12-15 23:42:33 -08002754 p->prev->next = skb;
2755 p->prev = skb;
2756 skb_header_release(skb);
2757
Herbert Xu5d38a072009-01-04 16:13:40 -08002758done:
2759 NAPI_GRO_CB(p)->count++;
Herbert Xu37fe4732009-01-17 19:48:13 +00002760 p->data_len += len;
2761 p->truesize += len;
2762 p->len += len;
Herbert Xu71d93b32008-12-15 23:42:33 -08002763
2764 NAPI_GRO_CB(skb)->same_flow = 1;
2765 return 0;
2766}
2767EXPORT_SYMBOL_GPL(skb_gro_receive);
2768
Linus Torvalds1da177e2005-04-16 15:20:36 -07002769void __init skb_init(void)
2770{
2771 skbuff_head_cache = kmem_cache_create("skbuff_head_cache",
2772 sizeof(struct sk_buff),
2773 0,
Alexey Dobriyane5d679f332006-08-26 19:25:52 -07002774 SLAB_HWCACHE_ALIGN|SLAB_PANIC,
Paul Mundt20c2df82007-07-20 10:11:58 +09002775 NULL);
David S. Millerd179cd12005-08-17 14:57:30 -07002776 skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache",
2777 (2*sizeof(struct sk_buff)) +
2778 sizeof(atomic_t),
2779 0,
Alexey Dobriyane5d679f332006-08-26 19:25:52 -07002780 SLAB_HWCACHE_ALIGN|SLAB_PANIC,
Paul Mundt20c2df82007-07-20 10:11:58 +09002781 NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002782}
2783
David Howells716ea3a2007-04-02 20:19:53 -07002784/**
2785 * skb_to_sgvec - Fill a scatter-gather list from a socket buffer
2786 * @skb: Socket buffer containing the buffers to be mapped
2787 * @sg: The scatter-gather list to map into
2788 * @offset: The offset into the buffer's contents to start mapping
2789 * @len: Length of buffer space to be mapped
2790 *
2791 * Fill the specified scatter-gather list with mappings/pointers into a
2792 * region of the buffer space attached to a socket buffer.
2793 */
David S. Miller51c739d2007-10-30 21:29:29 -07002794static int
2795__skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len)
David Howells716ea3a2007-04-02 20:19:53 -07002796{
David S. Miller1a028e52007-04-27 15:21:23 -07002797 int start = skb_headlen(skb);
2798 int i, copy = start - offset;
David S. Millerfbb398a2009-06-09 00:18:59 -07002799 struct sk_buff *frag_iter;
David Howells716ea3a2007-04-02 20:19:53 -07002800 int elt = 0;
2801
2802 if (copy > 0) {
2803 if (copy > len)
2804 copy = len;
Jens Axboe642f1492007-10-24 11:20:47 +02002805 sg_set_buf(sg, skb->data + offset, copy);
David Howells716ea3a2007-04-02 20:19:53 -07002806 elt++;
2807 if ((len -= copy) == 0)
2808 return elt;
2809 offset += copy;
2810 }
2811
2812 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
David S. Miller1a028e52007-04-27 15:21:23 -07002813 int end;
David Howells716ea3a2007-04-02 20:19:53 -07002814
Ilpo Järvinen547b7922008-07-25 21:43:18 -07002815 WARN_ON(start > offset + len);
David S. Miller1a028e52007-04-27 15:21:23 -07002816
2817 end = start + skb_shinfo(skb)->frags[i].size;
David Howells716ea3a2007-04-02 20:19:53 -07002818 if ((copy = end - offset) > 0) {
2819 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2820
2821 if (copy > len)
2822 copy = len;
Jens Axboe642f1492007-10-24 11:20:47 +02002823 sg_set_page(&sg[elt], frag->page, copy,
2824 frag->page_offset+offset-start);
David Howells716ea3a2007-04-02 20:19:53 -07002825 elt++;
2826 if (!(len -= copy))
2827 return elt;
2828 offset += copy;
2829 }
David S. Miller1a028e52007-04-27 15:21:23 -07002830 start = end;
David Howells716ea3a2007-04-02 20:19:53 -07002831 }
2832
David S. Millerfbb398a2009-06-09 00:18:59 -07002833 skb_walk_frags(skb, frag_iter) {
2834 int end;
David Howells716ea3a2007-04-02 20:19:53 -07002835
David S. Millerfbb398a2009-06-09 00:18:59 -07002836 WARN_ON(start > offset + len);
David Howells716ea3a2007-04-02 20:19:53 -07002837
David S. Millerfbb398a2009-06-09 00:18:59 -07002838 end = start + frag_iter->len;
2839 if ((copy = end - offset) > 0) {
2840 if (copy > len)
2841 copy = len;
2842 elt += __skb_to_sgvec(frag_iter, sg+elt, offset - start,
2843 copy);
2844 if ((len -= copy) == 0)
2845 return elt;
2846 offset += copy;
David Howells716ea3a2007-04-02 20:19:53 -07002847 }
David S. Millerfbb398a2009-06-09 00:18:59 -07002848 start = end;
David Howells716ea3a2007-04-02 20:19:53 -07002849 }
2850 BUG_ON(len);
2851 return elt;
2852}
2853
David S. Miller51c739d2007-10-30 21:29:29 -07002854int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len)
2855{
2856 int nsg = __skb_to_sgvec(skb, sg, offset, len);
2857
Jens Axboec46f2332007-10-31 12:06:37 +01002858 sg_mark_end(&sg[nsg - 1]);
David S. Miller51c739d2007-10-30 21:29:29 -07002859
2860 return nsg;
2861}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08002862EXPORT_SYMBOL_GPL(skb_to_sgvec);
David S. Miller51c739d2007-10-30 21:29:29 -07002863
David Howells716ea3a2007-04-02 20:19:53 -07002864/**
2865 * skb_cow_data - Check that a socket buffer's data buffers are writable
2866 * @skb: The socket buffer to check.
2867 * @tailbits: Amount of trailing space to be added
2868 * @trailer: Returned pointer to the skb where the @tailbits space begins
2869 *
2870 * Make sure that the data buffers attached to a socket buffer are
2871 * writable. If they are not, private copies are made of the data buffers
2872 * and the socket buffer is set to use these instead.
2873 *
2874 * If @tailbits is given, make sure that there is space to write @tailbits
2875 * bytes of data beyond current end of socket buffer. @trailer will be
2876 * set to point to the skb in which this space begins.
2877 *
2878 * The number of scatterlist elements required to completely map the
2879 * COW'd and extended socket buffer will be returned.
2880 */
2881int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer)
2882{
2883 int copyflag;
2884 int elt;
2885 struct sk_buff *skb1, **skb_p;
2886
2887 /* If skb is cloned or its head is paged, reallocate
2888 * head pulling out all the pages (pages are considered not writable
2889 * at the moment even if they are anonymous).
2890 */
2891 if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) &&
2892 __pskb_pull_tail(skb, skb_pagelen(skb)-skb_headlen(skb)) == NULL)
2893 return -ENOMEM;
2894
2895 /* Easy case. Most of packets will go this way. */
David S. Millerfbb398a2009-06-09 00:18:59 -07002896 if (!skb_has_frags(skb)) {
David Howells716ea3a2007-04-02 20:19:53 -07002897 /* A little of trouble, not enough of space for trailer.
2898 * This should not happen, when stack is tuned to generate
2899 * good frames. OK, on miss we reallocate and reserve even more
2900 * space, 128 bytes is fair. */
2901
2902 if (skb_tailroom(skb) < tailbits &&
2903 pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC))
2904 return -ENOMEM;
2905
2906 /* Voila! */
2907 *trailer = skb;
2908 return 1;
2909 }
2910
2911 /* Misery. We are in troubles, going to mincer fragments... */
2912
2913 elt = 1;
2914 skb_p = &skb_shinfo(skb)->frag_list;
2915 copyflag = 0;
2916
2917 while ((skb1 = *skb_p) != NULL) {
2918 int ntail = 0;
2919
2920 /* The fragment is partially pulled by someone,
2921 * this can happen on input. Copy it and everything
2922 * after it. */
2923
2924 if (skb_shared(skb1))
2925 copyflag = 1;
2926
2927 /* If the skb is the last, worry about trailer. */
2928
2929 if (skb1->next == NULL && tailbits) {
2930 if (skb_shinfo(skb1)->nr_frags ||
David S. Millerfbb398a2009-06-09 00:18:59 -07002931 skb_has_frags(skb1) ||
David Howells716ea3a2007-04-02 20:19:53 -07002932 skb_tailroom(skb1) < tailbits)
2933 ntail = tailbits + 128;
2934 }
2935
2936 if (copyflag ||
2937 skb_cloned(skb1) ||
2938 ntail ||
2939 skb_shinfo(skb1)->nr_frags ||
David S. Millerfbb398a2009-06-09 00:18:59 -07002940 skb_has_frags(skb1)) {
David Howells716ea3a2007-04-02 20:19:53 -07002941 struct sk_buff *skb2;
2942
2943 /* Fuck, we are miserable poor guys... */
2944 if (ntail == 0)
2945 skb2 = skb_copy(skb1, GFP_ATOMIC);
2946 else
2947 skb2 = skb_copy_expand(skb1,
2948 skb_headroom(skb1),
2949 ntail,
2950 GFP_ATOMIC);
2951 if (unlikely(skb2 == NULL))
2952 return -ENOMEM;
2953
2954 if (skb1->sk)
2955 skb_set_owner_w(skb2, skb1->sk);
2956
2957 /* Looking around. Are we still alive?
2958 * OK, link new skb, drop old one */
2959
2960 skb2->next = skb1->next;
2961 *skb_p = skb2;
2962 kfree_skb(skb1);
2963 skb1 = skb2;
2964 }
2965 elt++;
2966 *trailer = skb1;
2967 skb_p = &skb1->next;
2968 }
2969
2970 return elt;
2971}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08002972EXPORT_SYMBOL_GPL(skb_cow_data);
David Howells716ea3a2007-04-02 20:19:53 -07002973
Patrick Ohlyac45f602009-02-12 05:03:37 +00002974void skb_tstamp_tx(struct sk_buff *orig_skb,
2975 struct skb_shared_hwtstamps *hwtstamps)
2976{
2977 struct sock *sk = orig_skb->sk;
2978 struct sock_exterr_skb *serr;
2979 struct sk_buff *skb;
2980 int err;
2981
2982 if (!sk)
2983 return;
2984
2985 skb = skb_clone(orig_skb, GFP_ATOMIC);
2986 if (!skb)
2987 return;
2988
2989 if (hwtstamps) {
2990 *skb_hwtstamps(skb) =
2991 *hwtstamps;
2992 } else {
2993 /*
2994 * no hardware time stamps available,
2995 * so keep the skb_shared_tx and only
2996 * store software time stamp
2997 */
2998 skb->tstamp = ktime_get_real();
2999 }
3000
3001 serr = SKB_EXT_ERR(skb);
3002 memset(serr, 0, sizeof(*serr));
3003 serr->ee.ee_errno = ENOMSG;
3004 serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING;
3005 err = sock_queue_err_skb(sk, skb);
3006 if (err)
3007 kfree_skb(skb);
3008}
3009EXPORT_SYMBOL_GPL(skb_tstamp_tx);
3010
3011
Rusty Russellf35d9d82008-02-04 23:49:54 -05003012/**
3013 * skb_partial_csum_set - set up and verify partial csum values for packet
3014 * @skb: the skb to set
3015 * @start: the number of bytes after skb->data to start checksumming.
3016 * @off: the offset from start to place the checksum.
3017 *
3018 * For untrusted partially-checksummed packets, we need to make sure the values
3019 * for skb->csum_start and skb->csum_offset are valid so we don't oops.
3020 *
3021 * This function checks and sets those values and skb->ip_summed: if this
3022 * returns false you should drop the packet.
3023 */
3024bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off)
3025{
Herbert Xu5ff8dda2009-06-04 01:22:01 +00003026 if (unlikely(start > skb_headlen(skb)) ||
3027 unlikely((int)start + off > skb_headlen(skb) - 2)) {
Rusty Russellf35d9d82008-02-04 23:49:54 -05003028 if (net_ratelimit())
3029 printk(KERN_WARNING
3030 "bad partial csum: csum=%u/%u len=%u\n",
Herbert Xu5ff8dda2009-06-04 01:22:01 +00003031 start, off, skb_headlen(skb));
Rusty Russellf35d9d82008-02-04 23:49:54 -05003032 return false;
3033 }
3034 skb->ip_summed = CHECKSUM_PARTIAL;
3035 skb->csum_start = skb_headroom(skb) + start;
3036 skb->csum_offset = off;
3037 return true;
3038}
David S. Millerb4ac530fc2009-02-10 02:09:24 -08003039EXPORT_SYMBOL_GPL(skb_partial_csum_set);
Rusty Russellf35d9d82008-02-04 23:49:54 -05003040
Ben Hutchings4497b072008-06-19 16:22:28 -07003041void __skb_warn_lro_forwarding(const struct sk_buff *skb)
3042{
3043 if (net_ratelimit())
3044 pr_warning("%s: received packets cannot be forwarded"
3045 " while LRO is enabled\n", skb->dev->name);
3046}
Ben Hutchings4497b072008-06-19 16:22:28 -07003047EXPORT_SYMBOL(__skb_warn_lro_forwarding);