Version:  2.0.40 2.2.26 2.4.37 3.2 3.3 3.4 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16 3.17 3.18

Linux/net/core/skbuff.c

  1 /*
  2  *      Routines having to do with the 'struct sk_buff' memory handlers.
  3  *
  4  *      Authors:        Alan Cox <alan@lxorguk.ukuu.org.uk>
  5  *                      Florian La Roche <rzsfl@rz.uni-sb.de>
  6  *
  7  *      Fixes:
  8  *              Alan Cox        :       Fixed the worst of the load
  9  *                                      balancer bugs.
 10  *              Dave Platt      :       Interrupt stacking fix.
 11  *      Richard Kooijman        :       Timestamp fixes.
 12  *              Alan Cox        :       Changed buffer format.
 13  *              Alan Cox        :       destructor hook for AF_UNIX etc.
 14  *              Linus Torvalds  :       Better skb_clone.
 15  *              Alan Cox        :       Added skb_copy.
 16  *              Alan Cox        :       Added all the changed routines Linus
 17  *                                      only put in the headers
 18  *              Ray VanTassle   :       Fixed --skb->lock in free
 19  *              Alan Cox        :       skb_copy copy arp field
 20  *              Andi Kleen      :       slabified it.
 21  *              Robert Olsson   :       Removed skb_head_pool
 22  *
 23  *      NOTE:
 24  *              The __skb_ routines should be called with interrupts
 25  *      disabled, or you better be *real* sure that the operation is atomic
 26  *      with respect to whatever list is being frobbed (e.g. via lock_sock()
 27  *      or via disabling bottom half handlers, etc).
 28  *
 29  *      This program is free software; you can redistribute it and/or
 30  *      modify it under the terms of the GNU General Public License
 31  *      as published by the Free Software Foundation; either version
 32  *      2 of the License, or (at your option) any later version.
 33  */
 34 
 35 /*
 36  *      The functions in this file will not compile correctly with gcc 2.4.x
 37  */
 38 
 39 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 40 
 41 #include <linux/module.h>
 42 #include <linux/types.h>
 43 #include <linux/kernel.h>
 44 #include <linux/kmemcheck.h>
 45 #include <linux/mm.h>
 46 #include <linux/interrupt.h>
 47 #include <linux/in.h>
 48 #include <linux/inet.h>
 49 #include <linux/slab.h>
 50 #include <linux/tcp.h>
 51 #include <linux/udp.h>
 52 #include <linux/netdevice.h>
 53 #ifdef CONFIG_NET_CLS_ACT
 54 #include <net/pkt_sched.h>
 55 #endif
 56 #include <linux/string.h>
 57 #include <linux/skbuff.h>
 58 #include <linux/splice.h>
 59 #include <linux/cache.h>
 60 #include <linux/rtnetlink.h>
 61 #include <linux/init.h>
 62 #include <linux/scatterlist.h>
 63 #include <linux/errqueue.h>
 64 #include <linux/prefetch.h>
 65 #include <linux/if_vlan.h>
 66 
 67 #include <net/protocol.h>
 68 #include <net/dst.h>
 69 #include <net/sock.h>
 70 #include <net/checksum.h>
 71 #include <net/ip6_checksum.h>
 72 #include <net/xfrm.h>
 73 
 74 #include <asm/uaccess.h>
 75 #include <trace/events/skb.h>
 76 #include <linux/highmem.h>
 77 
 78 struct kmem_cache *skbuff_head_cache __read_mostly;
 79 static struct kmem_cache *skbuff_fclone_cache __read_mostly;
 80 
 81 /**
 82  *      skb_panic - private function for out-of-line support
 83  *      @skb:   buffer
 84  *      @sz:    size
 85  *      @addr:  address
 86  *      @msg:   skb_over_panic or skb_under_panic
 87  *
 88  *      Out-of-line support for skb_put() and skb_push().
 89  *      Called via the wrapper skb_over_panic() or skb_under_panic().
 90  *      Keep out of line to prevent kernel bloat.
 91  *      __builtin_return_address is not used because it is not always reliable.
 92  */
 93 static void skb_panic(struct sk_buff *skb, unsigned int sz, void *addr,
 94                       const char msg[])
 95 {
 96         pr_emerg("%s: text:%p len:%d put:%d head:%p data:%p tail:%#lx end:%#lx dev:%s\n",
 97                  msg, addr, skb->len, sz, skb->head, skb->data,
 98                  (unsigned long)skb->tail, (unsigned long)skb->end,
 99                  skb->dev ? skb->dev->name : "<NULL>");
100         BUG();
101 }
102 
103 static void skb_over_panic(struct sk_buff *skb, unsigned int sz, void *addr)
104 {
105         skb_panic(skb, sz, addr, __func__);
106 }
107 
108 static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr)
109 {
110         skb_panic(skb, sz, addr, __func__);
111 }
112 
113 /*
114  * kmalloc_reserve is a wrapper around kmalloc_node_track_caller that tells
115  * the caller if emergency pfmemalloc reserves are being used. If it is and
116  * the socket is later found to be SOCK_MEMALLOC then PFMEMALLOC reserves
117  * may be used. Otherwise, the packet data may be discarded until enough
118  * memory is free
119  */
120 #define kmalloc_reserve(size, gfp, node, pfmemalloc) \
121          __kmalloc_reserve(size, gfp, node, _RET_IP_, pfmemalloc)
122 
123 static void *__kmalloc_reserve(size_t size, gfp_t flags, int node,
124                                unsigned long ip, bool *pfmemalloc)
125 {
126         void *obj;
127         bool ret_pfmemalloc = false;
128 
129         /*
130          * Try a regular allocation, when that fails and we're not entitled
131          * to the reserves, fail.
132          */
133         obj = kmalloc_node_track_caller(size,
134                                         flags | __GFP_NOMEMALLOC | __GFP_NOWARN,
135                                         node);
136         if (obj || !(gfp_pfmemalloc_allowed(flags)))
137                 goto out;
138 
139         /* Try again but now we are using pfmemalloc reserves */
140         ret_pfmemalloc = true;
141         obj = kmalloc_node_track_caller(size, flags, node);
142 
143 out:
144         if (pfmemalloc)
145                 *pfmemalloc = ret_pfmemalloc;
146 
147         return obj;
148 }
149 
150 /*      Allocate a new skbuff. We do this ourselves so we can fill in a few
151  *      'private' fields and also do memory statistics to find all the
152  *      [BEEP] leaks.
153  *
154  */
155 
156 struct sk_buff *__alloc_skb_head(gfp_t gfp_mask, int node)
157 {
158         struct sk_buff *skb;
159 
160         /* Get the HEAD */
161         skb = kmem_cache_alloc_node(skbuff_head_cache,
162                                     gfp_mask & ~__GFP_DMA, node);
163         if (!skb)
164                 goto out;
165 
166         /*
167          * Only clear those fields we need to clear, not those that we will
168          * actually initialise below. Hence, don't put any more fields after
169          * the tail pointer in struct sk_buff!
170          */
171         memset(skb, 0, offsetof(struct sk_buff, tail));
172         skb->head = NULL;
173         skb->truesize = sizeof(struct sk_buff);
174         atomic_set(&skb->users, 1);
175 
176         skb->mac_header = (typeof(skb->mac_header))~0U;
177 out:
178         return skb;
179 }
180 
181 /**
182  *      __alloc_skb     -       allocate a network buffer
183  *      @size: size to allocate
184  *      @gfp_mask: allocation mask
185  *      @flags: If SKB_ALLOC_FCLONE is set, allocate from fclone cache
186  *              instead of head cache and allocate a cloned (child) skb.
187  *              If SKB_ALLOC_RX is set, __GFP_MEMALLOC will be used for
188  *              allocations in case the data is required for writeback
189  *      @node: numa node to allocate memory on
190  *
191  *      Allocate a new &sk_buff. The returned buffer has no headroom and a
192  *      tail room of at least size bytes. The object has a reference count
193  *      of one. The return is the buffer. On a failure the return is %NULL.
194  *
195  *      Buffers may only be allocated from interrupts using a @gfp_mask of
196  *      %GFP_ATOMIC.
197  */
198 struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
199                             int flags, int node)
200 {
201         struct kmem_cache *cache;
202         struct skb_shared_info *shinfo;
203         struct sk_buff *skb;
204         u8 *data;
205         bool pfmemalloc;
206 
207         cache = (flags & SKB_ALLOC_FCLONE)
208                 ? skbuff_fclone_cache : skbuff_head_cache;
209 
210         if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX))
211                 gfp_mask |= __GFP_MEMALLOC;
212 
213         /* Get the HEAD */
214         skb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node);
215         if (!skb)
216                 goto out;
217         prefetchw(skb);
218 
219         /* We do our best to align skb_shared_info on a separate cache
220          * line. It usually works because kmalloc(X > SMP_CACHE_BYTES) gives
221          * aligned memory blocks, unless SLUB/SLAB debug is enabled.
222          * Both skb->head and skb_shared_info are cache line aligned.
223          */
224         size = SKB_DATA_ALIGN(size);
225         size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
226         data = kmalloc_reserve(size, gfp_mask, node, &pfmemalloc);
227         if (!data)
228                 goto nodata;
229         /* kmalloc(size) might give us more room than requested.
230          * Put skb_shared_info exactly at the end of allocated zone,
231          * to allow max possible filling before reallocation.
232          */
233         size = SKB_WITH_OVERHEAD(ksize(data));
234         prefetchw(data + size);
235 
236         /*
237          * Only clear those fields we need to clear, not those that we will
238          * actually initialise below. Hence, don't put any more fields after
239          * the tail pointer in struct sk_buff!
240          */
241         memset(skb, 0, offsetof(struct sk_buff, tail));
242         /* Account for allocated memory : skb + skb->head */
243         skb->truesize = SKB_TRUESIZE(size);
244         skb->pfmemalloc = pfmemalloc;
245         atomic_set(&skb->users, 1);
246         skb->head = data;
247         skb->data = data;
248         skb_reset_tail_pointer(skb);
249         skb->end = skb->tail + size;
250         skb->mac_header = (typeof(skb->mac_header))~0U;
251         skb->transport_header = (typeof(skb->transport_header))~0U;
252 
253         /* make sure we initialize shinfo sequentially */
254         shinfo = skb_shinfo(skb);
255         memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
256         atomic_set(&shinfo->dataref, 1);
257         kmemcheck_annotate_variable(shinfo->destructor_arg);
258 
259         if (flags & SKB_ALLOC_FCLONE) {
260                 struct sk_buff_fclones *fclones;
261 
262                 fclones = container_of(skb, struct sk_buff_fclones, skb1);
263 
264                 kmemcheck_annotate_bitfield(&fclones->skb2, flags1);
265                 skb->fclone = SKB_FCLONE_ORIG;
266                 atomic_set(&fclones->fclone_ref, 1);
267 
268                 fclones->skb2.fclone = SKB_FCLONE_FREE;
269                 fclones->skb2.pfmemalloc = pfmemalloc;
270         }
271 out:
272         return skb;
273 nodata:
274         kmem_cache_free(cache, skb);
275         skb = NULL;
276         goto out;
277 }
278 EXPORT_SYMBOL(__alloc_skb);
279 
280 /**
281  * build_skb - build a network buffer
282  * @data: data buffer provided by caller
283  * @frag_size: size of fragment, or 0 if head was kmalloced
284  *
285  * Allocate a new &sk_buff. Caller provides space holding head and
286  * skb_shared_info. @data must have been allocated by kmalloc() only if
287  * @frag_size is 0, otherwise data should come from the page allocator.
288  * The return is the new skb buffer.
289  * On a failure the return is %NULL, and @data is not freed.
290  * Notes :
291  *  Before IO, driver allocates only data buffer where NIC put incoming frame
292  *  Driver should add room at head (NET_SKB_PAD) and
293  *  MUST add room at tail (SKB_DATA_ALIGN(skb_shared_info))
294  *  After IO, driver calls build_skb(), to allocate sk_buff and populate it
295  *  before giving packet to stack.
296  *  RX rings only contains data buffers, not full skbs.
297  */
298 struct sk_buff *build_skb(void *data, unsigned int frag_size)
299 {
300         struct skb_shared_info *shinfo;
301         struct sk_buff *skb;
302         unsigned int size = frag_size ? : ksize(data);
303 
304         skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC);
305         if (!skb)
306                 return NULL;
307 
308         size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
309 
310         memset(skb, 0, offsetof(struct sk_buff, tail));
311         skb->truesize = SKB_TRUESIZE(size);
312         skb->head_frag = frag_size != 0;
313         atomic_set(&skb->users, 1);
314         skb->head = data;
315         skb->data = data;
316         skb_reset_tail_pointer(skb);
317         skb->end = skb->tail + size;
318         skb->mac_header = (typeof(skb->mac_header))~0U;
319         skb->transport_header = (typeof(skb->transport_header))~0U;
320 
321         /* make sure we initialize shinfo sequentially */
322         shinfo = skb_shinfo(skb);
323         memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
324         atomic_set(&shinfo->dataref, 1);
325         kmemcheck_annotate_variable(shinfo->destructor_arg);
326 
327         return skb;
328 }
329 EXPORT_SYMBOL(build_skb);
330 
331 struct netdev_alloc_cache {
332         struct page_frag        frag;
333         /* we maintain a pagecount bias, so that we dont dirty cache line
334          * containing page->_count every time we allocate a fragment.
335          */
336         unsigned int            pagecnt_bias;
337 };
338 static DEFINE_PER_CPU(struct netdev_alloc_cache, netdev_alloc_cache);
339 
340 static void *__netdev_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
341 {
342         struct netdev_alloc_cache *nc;
343         void *data = NULL;
344         int order;
345         unsigned long flags;
346 
347         local_irq_save(flags);
348         nc = this_cpu_ptr(&netdev_alloc_cache);
349         if (unlikely(!nc->frag.page)) {
350 refill:
351                 for (order = NETDEV_FRAG_PAGE_MAX_ORDER; ;) {
352                         gfp_t gfp = gfp_mask;
353 
354                         if (order)
355                                 gfp |= __GFP_COMP | __GFP_NOWARN;
356                         nc->frag.page = alloc_pages(gfp, order);
357                         if (likely(nc->frag.page))
358                                 break;
359                         if (--order < 0)
360                                 goto end;
361                 }
362                 nc->frag.size = PAGE_SIZE << order;
363                 /* Even if we own the page, we do not use atomic_set().
364                  * This would break get_page_unless_zero() users.
365                  */
366                 atomic_add(NETDEV_PAGECNT_MAX_BIAS - 1,
367                            &nc->frag.page->_count);
368                 nc->pagecnt_bias = NETDEV_PAGECNT_MAX_BIAS;
369                 nc->frag.offset = 0;
370         }
371 
372         if (nc->frag.offset + fragsz > nc->frag.size) {
373                 if (atomic_read(&nc->frag.page->_count) != nc->pagecnt_bias) {
374                         if (!atomic_sub_and_test(nc->pagecnt_bias,
375                                                  &nc->frag.page->_count))
376                                 goto refill;
377                         /* OK, page count is 0, we can safely set it */
378                         atomic_set(&nc->frag.page->_count,
379                                    NETDEV_PAGECNT_MAX_BIAS);
380                 } else {
381                         atomic_add(NETDEV_PAGECNT_MAX_BIAS - nc->pagecnt_bias,
382                                    &nc->frag.page->_count);
383                 }
384                 nc->pagecnt_bias = NETDEV_PAGECNT_MAX_BIAS;
385                 nc->frag.offset = 0;
386         }
387 
388         data = page_address(nc->frag.page) + nc->frag.offset;
389         nc->frag.offset += fragsz;
390         nc->pagecnt_bias--;
391 end:
392         local_irq_restore(flags);
393         return data;
394 }
395 
396 /**
397  * netdev_alloc_frag - allocate a page fragment
398  * @fragsz: fragment size
399  *
400  * Allocates a frag from a page for receive buffer.
401  * Uses GFP_ATOMIC allocations.
402  */
403 void *netdev_alloc_frag(unsigned int fragsz)
404 {
405         return __netdev_alloc_frag(fragsz, GFP_ATOMIC | __GFP_COLD);
406 }
407 EXPORT_SYMBOL(netdev_alloc_frag);
408 
409 /**
410  *      __netdev_alloc_skb - allocate an skbuff for rx on a specific device
411  *      @dev: network device to receive on
412  *      @length: length to allocate
413  *      @gfp_mask: get_free_pages mask, passed to alloc_skb
414  *
415  *      Allocate a new &sk_buff and assign it a usage count of one. The
416  *      buffer has unspecified headroom built in. Users should allocate
417  *      the headroom they think they need without accounting for the
418  *      built in space. The built in space is used for optimisations.
419  *
420  *      %NULL is returned if there is no free memory.
421  */
422 struct sk_buff *__netdev_alloc_skb(struct net_device *dev,
423                                    unsigned int length, gfp_t gfp_mask)
424 {
425         struct sk_buff *skb = NULL;
426         unsigned int fragsz = SKB_DATA_ALIGN(length + NET_SKB_PAD) +
427                               SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
428 
429         if (fragsz <= PAGE_SIZE && !(gfp_mask & (__GFP_WAIT | GFP_DMA))) {
430                 void *data;
431 
432                 if (sk_memalloc_socks())
433                         gfp_mask |= __GFP_MEMALLOC;
434 
435                 data = __netdev_alloc_frag(fragsz, gfp_mask);
436 
437                 if (likely(data)) {
438                         skb = build_skb(data, fragsz);
439                         if (unlikely(!skb))
440                                 put_page(virt_to_head_page(data));
441                 }
442         } else {
443                 skb = __alloc_skb(length + NET_SKB_PAD, gfp_mask,
444                                   SKB_ALLOC_RX, NUMA_NO_NODE);
445         }
446         if (likely(skb)) {
447                 skb_reserve(skb, NET_SKB_PAD);
448                 skb->dev = dev;
449         }
450         return skb;
451 }
452 EXPORT_SYMBOL(__netdev_alloc_skb);
453 
454 void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off,
455                      int size, unsigned int truesize)
456 {
457         skb_fill_page_desc(skb, i, page, off, size);
458         skb->len += size;
459         skb->data_len += size;
460         skb->truesize += truesize;
461 }
462 EXPORT_SYMBOL(skb_add_rx_frag);
463 
464 void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size,
465                           unsigned int truesize)
466 {
467         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
468 
469         skb_frag_size_add(frag, size);
470         skb->len += size;
471         skb->data_len += size;
472         skb->truesize += truesize;
473 }
474 EXPORT_SYMBOL(skb_coalesce_rx_frag);
475 
476 static void skb_drop_list(struct sk_buff **listp)
477 {
478         kfree_skb_list(*listp);
479         *listp = NULL;
480 }
481 
482 static inline void skb_drop_fraglist(struct sk_buff *skb)
483 {
484         skb_drop_list(&skb_shinfo(skb)->frag_list);
485 }
486 
487 static void skb_clone_fraglist(struct sk_buff *skb)
488 {
489         struct sk_buff *list;
490 
491         skb_walk_frags(skb, list)
492                 skb_get(list);
493 }
494 
495 static void skb_free_head(struct sk_buff *skb)
496 {
497         if (skb->head_frag)
498                 put_page(virt_to_head_page(skb->head));
499         else
500                 kfree(skb->head);
501 }
502 
503 static void skb_release_data(struct sk_buff *skb)
504 {
505         struct skb_shared_info *shinfo = skb_shinfo(skb);
506         int i;
507 
508         if (skb->cloned &&
509             atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1,
510                               &shinfo->dataref))
511                 return;
512 
513         for (i = 0; i < shinfo->nr_frags; i++)
514                 __skb_frag_unref(&shinfo->frags[i]);
515 
516         /*
517          * If skb buf is from userspace, we need to notify the caller
518          * the lower device DMA has done;
519          */
520         if (shinfo->tx_flags & SKBTX_DEV_ZEROCOPY) {
521                 struct ubuf_info *uarg;
522 
523                 uarg = shinfo->destructor_arg;
524                 if (uarg->callback)
525                         uarg->callback(uarg, true);
526         }
527 
528         if (shinfo->frag_list)
529                 kfree_skb_list(shinfo->frag_list);
530 
531         skb_free_head(skb);
532 }
533 
534 /*
535  *      Free an skbuff by memory without cleaning the state.
536  */
537 static void kfree_skbmem(struct sk_buff *skb)
538 {
539         struct sk_buff_fclones *fclones;
540 
541         switch (skb->fclone) {
542         case SKB_FCLONE_UNAVAILABLE:
543                 kmem_cache_free(skbuff_head_cache, skb);
544                 break;
545 
546         case SKB_FCLONE_ORIG:
547                 fclones = container_of(skb, struct sk_buff_fclones, skb1);
548                 if (atomic_dec_and_test(&fclones->fclone_ref))
549                         kmem_cache_free(skbuff_fclone_cache, fclones);
550                 break;
551 
552         case SKB_FCLONE_CLONE:
553                 fclones = container_of(skb, struct sk_buff_fclones, skb2);
554 
555                 /* The clone portion is available for
556                  * fast-cloning again.
557                  */
558                 skb->fclone = SKB_FCLONE_FREE;
559 
560                 if (atomic_dec_and_test(&fclones->fclone_ref))
561                         kmem_cache_free(skbuff_fclone_cache, fclones);
562                 break;
563         }
564 }
565 
566 static void skb_release_head_state(struct sk_buff *skb)
567 {
568         skb_dst_drop(skb);
569 #ifdef CONFIG_XFRM
570         secpath_put(skb->sp);
571 #endif
572         if (skb->destructor) {
573                 WARN_ON(in_irq());
574                 skb->destructor(skb);
575         }
576 #if IS_ENABLED(CONFIG_NF_CONNTRACK)
577         nf_conntrack_put(skb->nfct);
578 #endif
579 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
580         nf_bridge_put(skb->nf_bridge);
581 #endif
582 /* XXX: IS this still necessary? - JHS */
583 #ifdef CONFIG_NET_SCHED
584         skb->tc_index = 0;
585 #ifdef CONFIG_NET_CLS_ACT
586         skb->tc_verd = 0;
587 #endif
588 #endif
589 }
590 
591 /* Free everything but the sk_buff shell. */
592 static void skb_release_all(struct sk_buff *skb)
593 {
594         skb_release_head_state(skb);
595         if (likely(skb->head))
596                 skb_release_data(skb);
597 }
598 
599 /**
600  *      __kfree_skb - private function
601  *      @skb: buffer
602  *
603  *      Free an sk_buff. Release anything attached to the buffer.
604  *      Clean the state. This is an internal helper function. Users should
605  *      always call kfree_skb
606  */
607 
608 void __kfree_skb(struct sk_buff *skb)
609 {
610         skb_release_all(skb);
611         kfree_skbmem(skb);
612 }
613 EXPORT_SYMBOL(__kfree_skb);
614 
615 /**
616  *      kfree_skb - free an sk_buff
617  *      @skb: buffer to free
618  *
619  *      Drop a reference to the buffer and free it if the usage count has
620  *      hit zero.
621  */
622 void kfree_skb(struct sk_buff *skb)
623 {
624         if (unlikely(!skb))
625                 return;
626         if (likely(atomic_read(&skb->users) == 1))
627                 smp_rmb();
628         else if (likely(!atomic_dec_and_test(&skb->users)))
629                 return;
630         trace_kfree_skb(skb, __builtin_return_address(0));
631         __kfree_skb(skb);
632 }
633 EXPORT_SYMBOL(kfree_skb);
634 
635 void kfree_skb_list(struct sk_buff *segs)
636 {
637         while (segs) {
638                 struct sk_buff *next = segs->next;
639 
640                 kfree_skb(segs);
641                 segs = next;
642         }
643 }
644 EXPORT_SYMBOL(kfree_skb_list);
645 
646 /**
647  *      skb_tx_error - report an sk_buff xmit error
648  *      @skb: buffer that triggered an error
649  *
650  *      Report xmit error if a device callback is tracking this skb.
651  *      skb must be freed afterwards.
652  */
653 void skb_tx_error(struct sk_buff *skb)
654 {
655         if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) {
656                 struct ubuf_info *uarg;
657 
658                 uarg = skb_shinfo(skb)->destructor_arg;
659                 if (uarg->callback)
660                         uarg->callback(uarg, false);
661                 skb_shinfo(skb)->tx_flags &= ~SKBTX_DEV_ZEROCOPY;
662         }
663 }
664 EXPORT_SYMBOL(skb_tx_error);
665 
666 /**
667  *      consume_skb - free an skbuff
668  *      @skb: buffer to free
669  *
670  *      Drop a ref to the buffer and free it if the usage count has hit zero
671  *      Functions identically to kfree_skb, but kfree_skb assumes that the frame
672  *      is being dropped after a failure and notes that
673  */
674 void consume_skb(struct sk_buff *skb)
675 {
676         if (unlikely(!skb))
677                 return;
678         if (likely(atomic_read(&skb->users) == 1))
679                 smp_rmb();
680         else if (likely(!atomic_dec_and_test(&skb->users)))
681                 return;
682         trace_consume_skb(skb);
683         __kfree_skb(skb);
684 }
685 EXPORT_SYMBOL(consume_skb);
686 
687 /* Make sure a field is enclosed inside headers_start/headers_end section */
688 #define CHECK_SKB_FIELD(field) \
689         BUILD_BUG_ON(offsetof(struct sk_buff, field) <          \
690                      offsetof(struct sk_buff, headers_start));  \
691         BUILD_BUG_ON(offsetof(struct sk_buff, field) >          \
692                      offsetof(struct sk_buff, headers_end));    \
693 
694 static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
695 {
696         new->tstamp             = old->tstamp;
697         /* We do not copy old->sk */
698         new->dev                = old->dev;
699         memcpy(new->cb, old->cb, sizeof(old->cb));
700         skb_dst_copy(new, old);
701 #ifdef CONFIG_XFRM
702         new->sp                 = secpath_get(old->sp);
703 #endif
704         __nf_copy(new, old, false);
705 
706         /* Note : this field could be in headers_start/headers_end section
707          * It is not yet because we do not want to have a 16 bit hole
708          */
709         new->queue_mapping = old->queue_mapping;
710 
711         memcpy(&new->headers_start, &old->headers_start,
712                offsetof(struct sk_buff, headers_end) -
713                offsetof(struct sk_buff, headers_start));
714         CHECK_SKB_FIELD(protocol);
715         CHECK_SKB_FIELD(csum);
716         CHECK_SKB_FIELD(hash);
717         CHECK_SKB_FIELD(priority);
718         CHECK_SKB_FIELD(skb_iif);
719         CHECK_SKB_FIELD(vlan_proto);
720         CHECK_SKB_FIELD(vlan_tci);
721         CHECK_SKB_FIELD(transport_header);
722         CHECK_SKB_FIELD(network_header);
723         CHECK_SKB_FIELD(mac_header);
724         CHECK_SKB_FIELD(inner_protocol);
725         CHECK_SKB_FIELD(inner_transport_header);
726         CHECK_SKB_FIELD(inner_network_header);
727         CHECK_SKB_FIELD(inner_mac_header);
728         CHECK_SKB_FIELD(mark);
729 #ifdef CONFIG_NETWORK_SECMARK
730         CHECK_SKB_FIELD(secmark);
731 #endif
732 #ifdef CONFIG_NET_RX_BUSY_POLL
733         CHECK_SKB_FIELD(napi_id);
734 #endif
735 #ifdef CONFIG_NET_SCHED
736         CHECK_SKB_FIELD(tc_index);
737 #ifdef CONFIG_NET_CLS_ACT
738         CHECK_SKB_FIELD(tc_verd);
739 #endif
740 #endif
741 
742 }
743 
744 /*
745  * You should not add any new code to this function.  Add it to
746  * __copy_skb_header above instead.
747  */
748 static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb)
749 {
750 #define C(x) n->x = skb->x
751 
752         n->next = n->prev = NULL;
753         n->sk = NULL;
754         __copy_skb_header(n, skb);
755 
756         C(len);
757         C(data_len);
758         C(mac_len);
759         n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len;
760         n->cloned = 1;
761         n->nohdr = 0;
762         n->destructor = NULL;
763         C(tail);
764         C(end);
765         C(head);
766         C(head_frag);
767         C(data);
768         C(truesize);
769         atomic_set(&n->users, 1);
770 
771         atomic_inc(&(skb_shinfo(skb)->dataref));
772         skb->cloned = 1;
773 
774         return n;
775 #undef C
776 }
777 
778 /**
779  *      skb_morph       -       morph one skb into another
780  *      @dst: the skb to receive the contents
781  *      @src: the skb to supply the contents
782  *
783  *      This is identical to skb_clone except that the target skb is
784  *      supplied by the user.
785  *
786  *      The target skb is returned upon exit.
787  */
788 struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src)
789 {
790         skb_release_all(dst);
791         return __skb_clone(dst, src);
792 }
793 EXPORT_SYMBOL_GPL(skb_morph);
794 
795 /**
796  *      skb_copy_ubufs  -       copy userspace skb frags buffers to kernel
797  *      @skb: the skb to modify
798  *      @gfp_mask: allocation priority
799  *
800  *      This must be called on SKBTX_DEV_ZEROCOPY skb.
801  *      It will copy all frags into kernel and drop the reference
802  *      to userspace pages.
803  *
804  *      If this function is called from an interrupt gfp_mask() must be
805  *      %GFP_ATOMIC.
806  *
807  *      Returns 0 on success or a negative error code on failure
808  *      to allocate kernel memory to copy to.
809  */
810 int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask)
811 {
812         int i;
813         int num_frags = skb_shinfo(skb)->nr_frags;
814         struct page *page, *head = NULL;
815         struct ubuf_info *uarg = skb_shinfo(skb)->destructor_arg;
816 
817         for (i = 0; i < num_frags; i++) {
818                 u8 *vaddr;
819                 skb_frag_t *f = &skb_shinfo(skb)->frags[i];
820 
821                 page = alloc_page(gfp_mask);
822                 if (!page) {
823                         while (head) {
824                                 struct page *next = (struct page *)page_private(head);
825                                 put_page(head);
826                                 head = next;
827                         }
828                         return -ENOMEM;
829                 }
830                 vaddr = kmap_atomic(skb_frag_page(f));
831                 memcpy(page_address(page),
832                        vaddr + f->page_offset, skb_frag_size(f));
833                 kunmap_atomic(vaddr);
834                 set_page_private(page, (unsigned long)head);
835                 head = page;
836         }
837 
838         /* skb frags release userspace buffers */
839         for (i = 0; i < num_frags; i++)
840                 skb_frag_unref(skb, i);
841 
842         uarg->callback(uarg, false);
843 
844         /* skb frags point to kernel buffers */
845         for (i = num_frags - 1; i >= 0; i--) {
846                 __skb_fill_page_desc(skb, i, head, 0,
847                                      skb_shinfo(skb)->frags[i].size);
848                 head = (struct page *)page_private(head);
849         }
850 
851         skb_shinfo(skb)->tx_flags &= ~SKBTX_DEV_ZEROCOPY;
852         return 0;
853 }
854 EXPORT_SYMBOL_GPL(skb_copy_ubufs);
855 
856 /**
857  *      skb_clone       -       duplicate an sk_buff
858  *      @skb: buffer to clone
859  *      @gfp_mask: allocation priority
860  *
861  *      Duplicate an &sk_buff. The new one is not owned by a socket. Both
862  *      copies share the same packet data but not structure. The new
863  *      buffer has a reference count of 1. If the allocation fails the
864  *      function returns %NULL otherwise the new buffer is returned.
865  *
866  *      If this function is called from an interrupt gfp_mask() must be
867  *      %GFP_ATOMIC.
868  */
869 
870 struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask)
871 {
872         struct sk_buff_fclones *fclones = container_of(skb,
873                                                        struct sk_buff_fclones,
874                                                        skb1);
875         struct sk_buff *n = &fclones->skb2;
876 
877         if (skb_orphan_frags(skb, gfp_mask))
878                 return NULL;
879 
880         if (skb->fclone == SKB_FCLONE_ORIG &&
881             n->fclone == SKB_FCLONE_FREE) {
882                 n->fclone = SKB_FCLONE_CLONE;
883                 atomic_inc(&fclones->fclone_ref);
884         } else {
885                 if (skb_pfmemalloc(skb))
886                         gfp_mask |= __GFP_MEMALLOC;
887 
888                 n = kmem_cache_alloc(skbuff_head_cache, gfp_mask);
889                 if (!n)
890                         return NULL;
891 
892                 kmemcheck_annotate_bitfield(n, flags1);
893                 n->fclone = SKB_FCLONE_UNAVAILABLE;
894         }
895 
896         return __skb_clone(n, skb);
897 }
898 EXPORT_SYMBOL(skb_clone);
899 
900 static void skb_headers_offset_update(struct sk_buff *skb, int off)
901 {
902         /* Only adjust this if it actually is csum_start rather than csum */
903         if (skb->ip_summed == CHECKSUM_PARTIAL)
904                 skb->csum_start += off;
905         /* {transport,network,mac}_header and tail are relative to skb->head */
906         skb->transport_header += off;
907         skb->network_header   += off;
908         if (skb_mac_header_was_set(skb))
909                 skb->mac_header += off;
910         skb->inner_transport_header += off;
911         skb->inner_network_header += off;
912         skb->inner_mac_header += off;
913 }
914 
915 static void copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
916 {
917         __copy_skb_header(new, old);
918 
919         skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size;
920         skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs;
921         skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type;
922 }
923 
924 static inline int skb_alloc_rx_flag(const struct sk_buff *skb)
925 {
926         if (skb_pfmemalloc(skb))
927                 return SKB_ALLOC_RX;
928         return 0;
929 }
930 
931 /**
932  *      skb_copy        -       create private copy of an sk_buff
933  *      @skb: buffer to copy
934  *      @gfp_mask: allocation priority
935  *
936  *      Make a copy of both an &sk_buff and its data. This is used when the
937  *      caller wishes to modify the data and needs a private copy of the
938  *      data to alter. Returns %NULL on failure or the pointer to the buffer
939  *      on success. The returned buffer has a reference count of 1.
940  *
941  *      As by-product this function converts non-linear &sk_buff to linear
942  *      one, so that &sk_buff becomes completely private and caller is allowed
943  *      to modify all the data of returned buffer. This means that this
944  *      function is not recommended for use in circumstances when only
945  *      header is going to be modified. Use pskb_copy() instead.
946  */
947 
948 struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask)
949 {
950         int headerlen = skb_headroom(skb);
951         unsigned int size = skb_end_offset(skb) + skb->data_len;
952         struct sk_buff *n = __alloc_skb(size, gfp_mask,
953                                         skb_alloc_rx_flag(skb), NUMA_NO_NODE);
954 
955         if (!n)
956                 return NULL;
957 
958         /* Set the data pointer */
959         skb_reserve(n, headerlen);
960         /* Set the tail pointer and length */
961         skb_put(n, skb->len);
962 
963         if (skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len))
964                 BUG();
965 
966         copy_skb_header(n, skb);
967         return n;
968 }
969 EXPORT_SYMBOL(skb_copy);
970 
971 /**
972  *      __pskb_copy_fclone      -  create copy of an sk_buff with private head.
973  *      @skb: buffer to copy
974  *      @headroom: headroom of new skb
975  *      @gfp_mask: allocation priority
976  *      @fclone: if true allocate the copy of the skb from the fclone
977  *      cache instead of the head cache; it is recommended to set this
978  *      to true for the cases where the copy will likely be cloned
979  *
980  *      Make a copy of both an &sk_buff and part of its data, located
981  *      in header. Fragmented data remain shared. This is used when
982  *      the caller wishes to modify only header of &sk_buff and needs
983  *      private copy of the header to alter. Returns %NULL on failure
984  *      or the pointer to the buffer on success.
985  *      The returned buffer has a reference count of 1.
986  */
987 
988 struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom,
989                                    gfp_t gfp_mask, bool fclone)
990 {
991         unsigned int size = skb_headlen(skb) + headroom;
992         int flags = skb_alloc_rx_flag(skb) | (fclone ? SKB_ALLOC_FCLONE : 0);
993         struct sk_buff *n = __alloc_skb(size, gfp_mask, flags, NUMA_NO_NODE);
994 
995         if (!n)
996                 goto out;
997 
998         /* Set the data pointer */
999         skb_reserve(n, headroom);
1000         /* Set the tail pointer and length */
1001         skb_put(n, skb_headlen(skb));
1002         /* Copy the bytes */
1003         skb_copy_from_linear_data(skb, n->data, n->len);
1004 
1005         n->truesize += skb->data_len;
1006         n->data_len  = skb->data_len;
1007         n->len       = skb->len;
1008 
1009         if (skb_shinfo(skb)->nr_frags) {
1010                 int i;
1011 
1012                 if (skb_orphan_frags(skb, gfp_mask)) {
1013                         kfree_skb(n);
1014                         n = NULL;
1015                         goto out;
1016                 }
1017                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1018                         skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i];
1019                         skb_frag_ref(skb, i);
1020                 }
1021                 skb_shinfo(n)->nr_frags = i;
1022         }
1023 
1024         if (skb_has_frag_list(skb)) {
1025                 skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list;
1026                 skb_clone_fraglist(n);
1027         }
1028 
1029         copy_skb_header(n, skb);
1030 out:
1031         return n;
1032 }
1033 EXPORT_SYMBOL(__pskb_copy_fclone);
1034 
1035 /**
1036  *      pskb_expand_head - reallocate header of &sk_buff
1037  *      @skb: buffer to reallocate
1038  *      @nhead: room to add at head
1039  *      @ntail: room to add at tail
1040  *      @gfp_mask: allocation priority
1041  *
1042  *      Expands (or creates identical copy, if @nhead and @ntail are zero)
1043  *      header of @skb. &sk_buff itself is not changed. &sk_buff MUST have
1044  *      reference count of 1. Returns zero in the case of success or error,
1045  *      if expansion failed. In the last case, &sk_buff is not changed.
1046  *
1047  *      All the pointers pointing into skb header may change and must be
1048  *      reloaded after call to this function.
1049  */
1050 
1051 int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
1052                      gfp_t gfp_mask)
1053 {
1054         int i;
1055         u8 *data;
1056         int size = nhead + skb_end_offset(skb) + ntail;
1057         long off;
1058 
1059         BUG_ON(nhead < 0);
1060 
1061         if (skb_shared(skb))
1062                 BUG();
1063 
1064         size = SKB_DATA_ALIGN(size);
1065 
1066         if (skb_pfmemalloc(skb))
1067                 gfp_mask |= __GFP_MEMALLOC;
1068         data = kmalloc_reserve(size + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
1069                                gfp_mask, NUMA_NO_NODE, NULL);
1070         if (!data)
1071                 goto nodata;
1072         size = SKB_WITH_OVERHEAD(ksize(data));
1073 
1074         /* Copy only real data... and, alas, header. This should be
1075          * optimized for the cases when header is void.
1076          */
1077         memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head);
1078 
1079         memcpy((struct skb_shared_info *)(data + size),
1080                skb_shinfo(skb),
1081                offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags]));
1082 
1083         /*
1084          * if shinfo is shared we must drop the old head gracefully, but if it
1085          * is not we can just drop the old head and let the existing refcount
1086          * be since all we did is relocate the values
1087          */
1088         if (skb_cloned(skb)) {
1089                 /* copy this zero copy skb frags */
1090                 if (skb_orphan_frags(skb, gfp_mask))
1091                         goto nofrags;
1092                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
1093                         skb_frag_ref(skb, i);
1094 
1095                 if (skb_has_frag_list(skb))
1096                         skb_clone_fraglist(skb);
1097 
1098                 skb_release_data(skb);
1099         } else {
1100                 skb_free_head(skb);
1101         }
1102         off = (data + nhead) - skb->head;
1103 
1104         skb->head     = data;
1105         skb->head_frag = 0;
1106         skb->data    += off;
1107 #ifdef NET_SKBUFF_DATA_USES_OFFSET
1108         skb->end      = size;
1109         off           = nhead;
1110 #else
1111         skb->end      = skb->head + size;
1112 #endif
1113         skb->tail             += off;
1114         skb_headers_offset_update(skb, nhead);
1115         skb->cloned   = 0;
1116         skb->hdr_len  = 0;
1117         skb->nohdr    = 0;
1118         atomic_set(&skb_shinfo(skb)->dataref, 1);
1119         return 0;
1120 
1121 nofrags:
1122         kfree(data);
1123 nodata:
1124         return -ENOMEM;
1125 }
1126 EXPORT_SYMBOL(pskb_expand_head);
1127 
1128 /* Make private copy of skb with writable head and some headroom */
1129 
1130 struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom)
1131 {
1132         struct sk_buff *skb2;
1133         int delta = headroom - skb_headroom(skb);
1134 
1135         if (delta <= 0)
1136                 skb2 = pskb_copy(skb, GFP_ATOMIC);
1137         else {
1138                 skb2 = skb_clone(skb, GFP_ATOMIC);
1139                 if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0,
1140                                              GFP_ATOMIC)) {
1141                         kfree_skb(skb2);
1142                         skb2 = NULL;
1143                 }
1144         }
1145         return skb2;
1146 }
1147 EXPORT_SYMBOL(skb_realloc_headroom);
1148 
1149 /**
1150  *      skb_copy_expand -       copy and expand sk_buff
1151  *      @skb: buffer to copy
1152  *      @newheadroom: new free bytes at head
1153  *      @newtailroom: new free bytes at tail
1154  *      @gfp_mask: allocation priority
1155  *
1156  *      Make a copy of both an &sk_buff and its data and while doing so
1157  *      allocate additional space.
1158  *
1159  *      This is used when the caller wishes to modify the data and needs a
1160  *      private copy of the data to alter as well as more space for new fields.
1161  *      Returns %NULL on failure or the pointer to the buffer
1162  *      on success. The returned buffer has a reference count of 1.
1163  *
1164  *      You must pass %GFP_ATOMIC as the allocation priority if this function
1165  *      is called from an interrupt.
1166  */
1167 struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
1168                                 int newheadroom, int newtailroom,
1169                                 gfp_t gfp_mask)
1170 {
1171         /*
1172          *      Allocate the copy buffer
1173          */
1174         struct sk_buff *n = __alloc_skb(newheadroom + skb->len + newtailroom,
1175                                         gfp_mask, skb_alloc_rx_flag(skb),
1176                                         NUMA_NO_NODE);
1177         int oldheadroom = skb_headroom(skb);
1178         int head_copy_len, head_copy_off;
1179 
1180         if (!n)
1181                 return NULL;
1182 
1183         skb_reserve(n, newheadroom);
1184 
1185         /* Set the tail pointer and length */
1186         skb_put(n, skb->len);
1187 
1188         head_copy_len = oldheadroom;
1189         head_copy_off = 0;
1190         if (newheadroom <= head_copy_len)
1191                 head_copy_len = newheadroom;
1192         else
1193                 head_copy_off = newheadroom - head_copy_len;
1194 
1195         /* Copy the linear header and data. */
1196         if (skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off,
1197                           skb->len + head_copy_len))
1198                 BUG();
1199 
1200         copy_skb_header(n, skb);
1201 
1202         skb_headers_offset_update(n, newheadroom - oldheadroom);
1203 
1204         return n;
1205 }
1206 EXPORT_SYMBOL(skb_copy_expand);
1207 
1208 /**
1209  *      skb_pad                 -       zero pad the tail of an skb
1210  *      @skb: buffer to pad
1211  *      @pad: space to pad
1212  *
1213  *      Ensure that a buffer is followed by a padding area that is zero
1214  *      filled. Used by network drivers which may DMA or transfer data
1215  *      beyond the buffer end onto the wire.
1216  *
1217  *      May return error in out of memory cases. The skb is freed on error.
1218  */
1219 
1220 int skb_pad(struct sk_buff *skb, int pad)
1221 {
1222         int err;
1223         int ntail;
1224 
1225         /* If the skbuff is non linear tailroom is always zero.. */
1226         if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) {
1227                 memset(skb->data+skb->len, 0, pad);
1228                 return 0;
1229         }
1230 
1231         ntail = skb->data_len + pad - (skb->end - skb->tail);
1232         if (likely(skb_cloned(skb) || ntail > 0)) {
1233                 err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC);
1234                 if (unlikely(err))
1235                         goto free_skb;
1236         }
1237 
1238         /* FIXME: The use of this function with non-linear skb's really needs
1239          * to be audited.
1240          */
1241         err = skb_linearize(skb);
1242         if (unlikely(err))
1243                 goto free_skb;
1244 
1245         memset(skb->data + skb->len, 0, pad);
1246         return 0;
1247 
1248 free_skb:
1249         kfree_skb(skb);
1250         return err;
1251 }
1252 EXPORT_SYMBOL(skb_pad);
1253 
1254 /**
1255  *      pskb_put - add data to the tail of a potentially fragmented buffer
1256  *      @skb: start of the buffer to use
1257  *      @tail: tail fragment of the buffer to use
1258  *      @len: amount of data to add
1259  *
1260  *      This function extends the used data area of the potentially
1261  *      fragmented buffer. @tail must be the last fragment of @skb -- or
1262  *      @skb itself. If this would exceed the total buffer size the kernel
1263  *      will panic. A pointer to the first byte of the extra data is
1264  *      returned.
1265  */
1266 
1267 unsigned char *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len)
1268 {
1269         if (tail != skb) {
1270                 skb->data_len += len;
1271                 skb->len += len;
1272         }
1273         return skb_put(tail, len);
1274 }
1275 EXPORT_SYMBOL_GPL(pskb_put);
1276 
1277 /**
1278  *      skb_put - add data to a buffer
1279  *      @skb: buffer to use
1280  *      @len: amount of data to add
1281  *
1282  *      This function extends the used data area of the buffer. If this would
1283  *      exceed the total buffer size the kernel will panic. A pointer to the
1284  *      first byte of the extra data is returned.
1285  */
1286 unsigned char *skb_put(struct sk_buff *skb, unsigned int len)
1287 {
1288         unsigned char *tmp = skb_tail_pointer(skb);
1289         SKB_LINEAR_ASSERT(skb);
1290         skb->tail += len;
1291         skb->len  += len;
1292         if (unlikely(skb->tail > skb->end))
1293                 skb_over_panic(skb, len, __builtin_return_address(0));
1294         return tmp;
1295 }
1296 EXPORT_SYMBOL(skb_put);
1297 
1298 /**
1299  *      skb_push - add data to the start of a buffer
1300  *      @skb: buffer to use
1301  *      @len: amount of data to add
1302  *
1303  *      This function extends the used data area of the buffer at the buffer
1304  *      start. If this would exceed the total buffer headroom the kernel will
1305  *      panic. A pointer to the first byte of the extra data is returned.
1306  */
1307 unsigned char *skb_push(struct sk_buff *skb, unsigned int len)
1308 {
1309         skb->data -= len;
1310         skb->len  += len;
1311         if (unlikely(skb->data<skb->head))
1312                 skb_under_panic(skb, len, __builtin_return_address(0));
1313         return skb->data;
1314 }
1315 EXPORT_SYMBOL(skb_push);
1316 
1317 /**
1318  *      skb_pull - remove data from the start of a buffer
1319  *      @skb: buffer to use
1320  *      @len: amount of data to remove
1321  *
1322  *      This function removes data from the start of a buffer, returning
1323  *      the memory to the headroom. A pointer to the next data in the buffer
1324  *      is returned. Once the data has been pulled future pushes will overwrite
1325  *      the old data.
1326  */
1327 unsigned char *skb_pull(struct sk_buff *skb, unsigned int len)
1328 {
1329         return skb_pull_inline(skb, len);
1330 }
1331 EXPORT_SYMBOL(skb_pull);
1332 
1333 /**
1334  *      skb_trim - remove end from a buffer
1335  *      @skb: buffer to alter
1336  *      @len: new length
1337  *
1338  *      Cut the length of a buffer down by removing data from the tail. If
1339  *      the buffer is already under the length specified it is not modified.
1340  *      The skb must be linear.
1341  */
1342 void skb_trim(struct sk_buff *skb, unsigned int len)
1343 {
1344         if (skb->len > len)
1345                 __skb_trim(skb, len);
1346 }
1347 EXPORT_SYMBOL(skb_trim);
1348 
1349 /* Trims skb to length len. It can change skb pointers.
1350  */
1351 
1352 int ___pskb_trim(struct sk_buff *skb, unsigned int len)
1353 {
1354         struct sk_buff **fragp;
1355         struct sk_buff *frag;
1356         int offset = skb_headlen(skb);
1357         int nfrags = skb_shinfo(skb)->nr_frags;
1358         int i;
1359         int err;
1360 
1361         if (skb_cloned(skb) &&
1362             unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC))))
1363                 return err;
1364 
1365         i = 0;
1366         if (offset >= len)
1367                 goto drop_pages;
1368 
1369         for (; i < nfrags; i++) {
1370                 int end = offset + skb_frag_size(&skb_shinfo(skb)->frags[i]);
1371 
1372                 if (end < len) {
1373                         offset = end;
1374                         continue;
1375                 }
1376 
1377                 skb_frag_size_set(&skb_shinfo(skb)->frags[i++], len - offset);
1378 
1379 drop_pages:
1380                 skb_shinfo(skb)->nr_frags = i;
1381 
1382                 for (; i < nfrags; i++)
1383                         skb_frag_unref(skb, i);
1384 
1385                 if (skb_has_frag_list(skb))
1386                         skb_drop_fraglist(skb);
1387                 goto done;
1388         }
1389 
1390         for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp);
1391              fragp = &frag->next) {
1392                 int end = offset + frag->len;
1393 
1394                 if (skb_shared(frag)) {
1395                         struct sk_buff *nfrag;
1396 
1397                         nfrag = skb_clone(frag, GFP_ATOMIC);
1398                         if (unlikely(!nfrag))
1399                                 return -ENOMEM;
1400 
1401                         nfrag->next = frag->next;
1402                         consume_skb(frag);
1403                         frag = nfrag;
1404                         *fragp = frag;
1405                 }
1406 
1407                 if (end < len) {
1408                         offset = end;
1409                         continue;
1410                 }
1411 
1412                 if (end > len &&
1413                     unlikely((err = pskb_trim(frag, len - offset))))
1414                         return err;
1415 
1416                 if (frag->next)
1417                         skb_drop_list(&frag->next);
1418                 break;
1419         }
1420 
1421 done:
1422         if (len > skb_headlen(skb)) {
1423                 skb->data_len -= skb->len - len;
1424                 skb->len       = len;
1425         } else {
1426                 skb->len       = len;
1427                 skb->data_len  = 0;
1428                 skb_set_tail_pointer(skb, len);
1429         }
1430 
1431         return 0;
1432 }
1433 EXPORT_SYMBOL(___pskb_trim);
1434 
1435 /**
1436  *      __pskb_pull_tail - advance tail of skb header
1437  *      @skb: buffer to reallocate
1438  *      @delta: number of bytes to advance tail
1439  *
1440  *      The function makes a sense only on a fragmented &sk_buff,
1441  *      it expands header moving its tail forward and copying necessary
1442  *      data from fragmented part.
1443  *
1444  *      &sk_buff MUST have reference count of 1.
1445  *
1446  *      Returns %NULL (and &sk_buff does not change) if pull failed
1447  *      or value of new tail of skb in the case of success.
1448  *
1449  *      All the pointers pointing into skb header may change and must be
1450  *      reloaded after call to this function.
1451  */
1452 
1453 /* Moves tail of skb head forward, copying data from fragmented part,
1454  * when it is necessary.
1455  * 1. It may fail due to malloc failure.
1456  * 2. It may change skb pointers.
1457  *
1458  * It is pretty complicated. Luckily, it is called only in exceptional cases.
1459  */
1460 unsigned char *__pskb_pull_tail(struct sk_buff *skb, int delta)
1461 {
1462         /* If skb has not enough free space at tail, get new one
1463          * plus 128 bytes for future expansions. If we have enough
1464          * room at tail, reallocate without expansion only if skb is cloned.
1465          */
1466         int i, k, eat = (skb->tail + delta) - skb->end;
1467 
1468         if (eat > 0 || skb_cloned(skb)) {
1469                 if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0,
1470                                      GFP_ATOMIC))
1471                         return NULL;
1472         }
1473 
1474         if (skb_copy_bits(skb, skb_headlen(skb), skb_tail_pointer(skb), delta))
1475                 BUG();
1476 
1477         /* Optimization: no fragments, no reasons to preestimate
1478          * size of pulled pages. Superb.
1479          */
1480         if (!skb_has_frag_list(skb))
1481                 goto pull_pages;
1482 
1483         /* Estimate size of pulled pages. */
1484         eat = delta;
1485         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1486                 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
1487 
1488                 if (size >= eat)
1489                         goto pull_pages;
1490                 eat -= size;
1491         }
1492 
1493         /* If we need update frag list, we are in troubles.
1494          * Certainly, it possible to add an offset to skb data,
1495          * but taking into account that pulling is expected to
1496          * be very rare operation, it is worth to fight against
1497          * further bloating skb head and crucify ourselves here instead.
1498          * Pure masohism, indeed. 8)8)
1499          */
1500         if (eat) {
1501                 struct sk_buff *list = skb_shinfo(skb)->frag_list;
1502                 struct sk_buff *clone = NULL;
1503                 struct sk_buff *insp = NULL;
1504 
1505                 do {
1506                         BUG_ON(!list);
1507 
1508                         if (list->len <= eat) {
1509                                 /* Eaten as whole. */
1510                                 eat -= list->len;
1511                                 list = list->next;
1512                                 insp = list;
1513                         } else {
1514                                 /* Eaten partially. */
1515 
1516                                 if (skb_shared(list)) {
1517                                         /* Sucks! We need to fork list. :-( */
1518                                         clone = skb_clone(list, GFP_ATOMIC);
1519                                         if (!clone)
1520                                                 return NULL;
1521                                         insp = list->next;
1522                                         list = clone;
1523                                 } else {
1524                                         /* This may be pulled without
1525                                          * problems. */
1526                                         insp = list;
1527                                 }
1528                                 if (!pskb_pull(list, eat)) {
1529                                         kfree_skb(clone);
1530                                         return NULL;
1531                                 }
1532                                 break;
1533                         }
1534                 } while (eat);
1535 
1536                 /* Free pulled out fragments. */
1537                 while ((list = skb_shinfo(skb)->frag_list) != insp) {
1538                         skb_shinfo(skb)->frag_list = list->next;
1539                         kfree_skb(list);
1540                 }
1541                 /* And insert new clone at head. */
1542                 if (clone) {
1543                         clone->next = list;
1544                         skb_shinfo(skb)->frag_list = clone;
1545                 }
1546         }
1547         /* Success! Now we may commit changes to skb data. */
1548 
1549 pull_pages:
1550         eat = delta;
1551         k = 0;
1552         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1553                 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
1554 
1555                 if (size <= eat) {
1556                         skb_frag_unref(skb, i);
1557                         eat -= size;
1558                 } else {
1559                         skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i];
1560                         if (eat) {
1561                                 skb_shinfo(skb)->frags[k].page_offset += eat;
1562                                 skb_frag_size_sub(&skb_shinfo(skb)->frags[k], eat);
1563                                 eat = 0;
1564                         }
1565                         k++;
1566                 }
1567         }
1568         skb_shinfo(skb)->nr_frags = k;
1569 
1570         skb->tail     += delta;
1571         skb->data_len -= delta;
1572 
1573         return skb_tail_pointer(skb);
1574 }
1575 EXPORT_SYMBOL(__pskb_pull_tail);
1576 
1577 /**
1578  *      skb_copy_bits - copy bits from skb to kernel buffer
1579  *      @skb: source skb
1580  *      @offset: offset in source
1581  *      @to: destination buffer
1582  *      @len: number of bytes to copy
1583  *
1584  *      Copy the specified number of bytes from the source skb to the
1585  *      destination buffer.
1586  *
1587  *      CAUTION ! :
1588  *              If its prototype is ever changed,
1589  *              check arch/{*}/net/{*}.S files,
1590  *              since it is called from BPF assembly code.
1591  */
1592 int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len)
1593 {
1594         int start = skb_headlen(skb);
1595         struct sk_buff *frag_iter;
1596         int i, copy;
1597 
1598         if (offset > (int)skb->len - len)
1599                 goto fault;
1600 
1601         /* Copy header. */
1602         if ((copy = start - offset) > 0) {
1603                 if (copy > len)
1604                         copy = len;
1605                 skb_copy_from_linear_data_offset(skb, offset, to, copy);
1606                 if ((len -= copy) == 0)
1607                         return 0;
1608                 offset += copy;
1609                 to     += copy;
1610         }
1611 
1612         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1613                 int end;
1614                 skb_frag_t *f = &skb_shinfo(skb)->frags[i];
1615 
1616                 WARN_ON(start > offset + len);
1617 
1618                 end = start + skb_frag_size(f);
1619                 if ((copy = end - offset) > 0) {
1620                         u8 *vaddr;
1621 
1622                         if (copy > len)
1623                                 copy = len;
1624 
1625                         vaddr = kmap_atomic(skb_frag_page(f));
1626                         memcpy(to,
1627                                vaddr + f->page_offset + offset - start,
1628                                copy);
1629                         kunmap_atomic(vaddr);
1630 
1631                         if ((len -= copy) == 0)
1632                                 return 0;
1633                         offset += copy;
1634                         to     += copy;
1635                 }
1636                 start = end;
1637         }
1638 
1639         skb_walk_frags(skb, frag_iter) {
1640                 int end;
1641 
1642                 WARN_ON(start > offset + len);
1643 
1644                 end = start + frag_iter->len;
1645                 if ((copy = end - offset) > 0) {
1646                         if (copy > len)
1647                                 copy = len;
1648                         if (skb_copy_bits(frag_iter, offset - start, to, copy))
1649                                 goto fault;
1650                         if ((len -= copy) == 0)
1651                                 return 0;
1652                         offset += copy;
1653                         to     += copy;
1654                 }
1655                 start = end;
1656         }
1657 
1658         if (!len)
1659                 return 0;
1660 
1661 fault:
1662         return -EFAULT;
1663 }
1664 EXPORT_SYMBOL(skb_copy_bits);
1665 
1666 /*
1667  * Callback from splice_to_pipe(), if we need to release some pages
1668  * at the end of the spd in case we error'ed out in filling the pipe.
1669  */
1670 static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i)
1671 {
1672         put_page(spd->pages[i]);
1673 }
1674 
1675 static struct page *linear_to_page(struct page *page, unsigned int *len,
1676                                    unsigned int *offset,
1677                                    struct sock *sk)
1678 {
1679         struct page_frag *pfrag = sk_page_frag(sk);
1680 
1681         if (!sk_page_frag_refill(sk, pfrag))
1682                 return NULL;
1683 
1684         *len = min_t(unsigned int, *len, pfrag->size - pfrag->offset);
1685 
1686         memcpy(page_address(pfrag->page) + pfrag->offset,
1687                page_address(page) + *offset, *len);
1688         *offset = pfrag->offset;
1689         pfrag->offset += *len;
1690 
1691         return pfrag->page;
1692 }
1693 
1694 static bool spd_can_coalesce(const struct splice_pipe_desc *spd,
1695                              struct page *page,
1696                              unsigned int offset)
1697 {
1698         return  spd->nr_pages &&
1699                 spd->pages[spd->nr_pages - 1] == page &&
1700                 (spd->partial[spd->nr_pages - 1].offset +
1701                  spd->partial[spd->nr_pages - 1].len == offset);
1702 }
1703 
1704 /*
1705  * Fill page/offset/length into spd, if it can hold more pages.
1706  */
1707 static bool spd_fill_page(struct splice_pipe_desc *spd,
1708                           struct pipe_inode_info *pipe, struct page *page,
1709                           unsigned int *len, unsigned int offset,
1710                           bool linear,
1711                           struct sock *sk)
1712 {
1713         if (unlikely(spd->nr_pages == MAX_SKB_FRAGS))
1714                 return true;
1715 
1716         if (linear) {
1717                 page = linear_to_page(page, len, &offset, sk);
1718                 if (!page)
1719                         return true;
1720         }
1721         if (spd_can_coalesce(spd, page, offset)) {
1722                 spd->partial[spd->nr_pages - 1].len += *len;
1723                 return false;
1724         }
1725         get_page(page);
1726         spd->pages[spd->nr_pages] = page;
1727         spd->partial[spd->nr_pages].len = *len;
1728         spd->partial[spd->nr_pages].offset = offset;
1729         spd->nr_pages++;
1730 
1731         return false;
1732 }
1733 
1734 static bool __splice_segment(struct page *page, unsigned int poff,
1735                              unsigned int plen, unsigned int *off,
1736                              unsigned int *len,
1737                              struct splice_pipe_desc *spd, bool linear,
1738                              struct sock *sk,
1739                              struct pipe_inode_info *pipe)
1740 {
1741         if (!*len)
1742                 return true;
1743 
1744         /* skip this segment if already processed */
1745         if (*off >= plen) {
1746                 *off -= plen;
1747                 return false;
1748         }
1749 
1750         /* ignore any bits we already processed */
1751         poff += *off;
1752         plen -= *off;
1753         *off = 0;
1754 
1755         do {
1756                 unsigned int flen = min(*len, plen);
1757 
1758                 if (spd_fill_page(spd, pipe, page, &flen, poff,
1759                                   linear, sk))
1760                         return true;
1761                 poff += flen;
1762                 plen -= flen;
1763                 *len -= flen;
1764         } while (*len && plen);
1765 
1766         return false;
1767 }
1768 
1769 /*
1770  * Map linear and fragment data from the skb to spd. It reports true if the
1771  * pipe is full or if we already spliced the requested length.
1772  */
1773 static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe,
1774                               unsigned int *offset, unsigned int *len,
1775                               struct splice_pipe_desc *spd, struct sock *sk)
1776 {
1777         int seg;
1778 
1779         /* map the linear part :
1780          * If skb->head_frag is set, this 'linear' part is backed by a
1781          * fragment, and if the head is not shared with any clones then
1782          * we can avoid a copy since we own the head portion of this page.
1783          */
1784         if (__splice_segment(virt_to_page(skb->data),
1785                              (unsigned long) skb->data & (PAGE_SIZE - 1),
1786                              skb_headlen(skb),
1787                              offset, len, spd,
1788                              skb_head_is_locked(skb),
1789                              sk, pipe))
1790                 return true;
1791 
1792         /*
1793          * then map the fragments
1794          */
1795         for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) {
1796                 const skb_frag_t *f = &skb_shinfo(skb)->frags[seg];
1797 
1798                 if (__splice_segment(skb_frag_page(f),
1799                                      f->page_offset, skb_frag_size(f),
1800                                      offset, len, spd, false, sk, pipe))
1801                         return true;
1802         }
1803 
1804         return false;
1805 }
1806 
1807 /*
1808  * Map data from the skb to a pipe. Should handle both the linear part,
1809  * the fragments, and the frag list. It does NOT handle frag lists within
1810  * the frag list, if such a thing exists. We'd probably need to recurse to
1811  * handle that cleanly.
1812  */
1813 int skb_splice_bits(struct sk_buff *skb, unsigned int offset,
1814                     struct pipe_inode_info *pipe, unsigned int tlen,
1815                     unsigned int flags)
1816 {
1817         struct partial_page partial[MAX_SKB_FRAGS];
1818         struct page *pages[MAX_SKB_FRAGS];
1819         struct splice_pipe_desc spd = {
1820                 .pages = pages,
1821                 .partial = partial,
1822                 .nr_pages_max = MAX_SKB_FRAGS,
1823                 .flags = flags,
1824                 .ops = &nosteal_pipe_buf_ops,
1825                 .spd_release = sock_spd_release,
1826         };
1827         struct sk_buff *frag_iter;
1828         struct sock *sk = skb->sk;
1829         int ret = 0;
1830 
1831         /*
1832          * __skb_splice_bits() only fails if the output has no room left,
1833          * so no point in going over the frag_list for the error case.
1834          */
1835         if (__skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk))
1836                 goto done;
1837         else if (!tlen)
1838                 goto done;
1839 
1840         /*
1841          * now see if we have a frag_list to map
1842          */
1843         skb_walk_frags(skb, frag_iter) {
1844                 if (!tlen)
1845                         break;
1846                 if (__skb_splice_bits(frag_iter, pipe, &offset, &tlen, &spd, sk))
1847                         break;
1848         }
1849 
1850 done:
1851         if (spd.nr_pages) {
1852                 /*
1853                  * Drop the socket lock, otherwise we have reverse
1854                  * locking dependencies between sk_lock and i_mutex
1855                  * here as compared to sendfile(). We enter here
1856                  * with the socket lock held, and splice_to_pipe() will
1857                  * grab the pipe inode lock. For sendfile() emulation,
1858                  * we call into ->sendpage() with the i_mutex lock held
1859                  * and networking will grab the socket lock.
1860                  */
1861                 release_sock(sk);
1862                 ret = splice_to_pipe(pipe, &spd);
1863                 lock_sock(sk);
1864         }
1865 
1866         return ret;
1867 }
1868 
1869 /**
1870  *      skb_store_bits - store bits from kernel buffer to skb
1871  *      @skb: destination buffer
1872  *      @offset: offset in destination
1873  *      @from: source buffer
1874  *      @len: number of bytes to copy
1875  *
1876  *      Copy the specified number of bytes from the source buffer to the
1877  *      destination skb.  This function handles all the messy bits of
1878  *      traversing fragment lists and such.
1879  */
1880 
1881 int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len)
1882 {
1883         int start = skb_headlen(skb);
1884         struct sk_buff *frag_iter;
1885         int i, copy;
1886 
1887         if (offset > (int)skb->len - len)
1888                 goto fault;
1889 
1890         if ((copy = start - offset) > 0) {
1891                 if (copy > len)
1892                         copy = len;
1893                 skb_copy_to_linear_data_offset(skb, offset, from, copy);
1894                 if ((len -= copy) == 0)
1895                         return 0;
1896                 offset += copy;
1897                 from += copy;
1898         }
1899 
1900         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1901                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1902                 int end;
1903 
1904                 WARN_ON(start > offset + len);
1905 
1906                 end = start + skb_frag_size(frag);
1907                 if ((copy = end - offset) > 0) {
1908                         u8 *vaddr;
1909 
1910                         if (copy > len)
1911                                 copy = len;
1912 
1913                         vaddr = kmap_atomic(skb_frag_page(frag));
1914                         memcpy(vaddr + frag->page_offset + offset - start,
1915                                from, copy);
1916                         kunmap_atomic(vaddr);
1917 
1918                         if ((len -= copy) == 0)
1919                                 return 0;
1920                         offset += copy;
1921                         from += copy;
1922                 }
1923                 start = end;
1924         }
1925 
1926         skb_walk_frags(skb, frag_iter) {
1927                 int end;
1928 
1929                 WARN_ON(start > offset + len);
1930 
1931                 end = start + frag_iter->len;
1932                 if ((copy = end - offset) > 0) {
1933                         if (copy > len)
1934                                 copy = len;
1935                         if (skb_store_bits(frag_iter, offset - start,
1936                                            from, copy))
1937                                 goto fault;
1938                         if ((len -= copy) == 0)
1939                                 return 0;
1940                         offset += copy;
1941                         from += copy;
1942                 }
1943                 start = end;
1944         }
1945         if (!len)
1946                 return 0;
1947 
1948 fault:
1949         return -EFAULT;
1950 }
1951 EXPORT_SYMBOL(skb_store_bits);
1952 
1953 /* Checksum skb data. */
1954 __wsum __skb_checksum(const struct sk_buff *skb, int offset, int len,
1955                       __wsum csum, const struct skb_checksum_ops *ops)
1956 {
1957         int start = skb_headlen(skb);
1958         int i, copy = start - offset;
1959         struct sk_buff *frag_iter;
1960         int pos = 0;
1961 
1962         /* Checksum header. */
1963         if (copy > 0) {
1964                 if (copy > len)
1965                         copy = len;
1966                 csum = ops->update(skb->data + offset, copy, csum);
1967                 if ((len -= copy) == 0)
1968                         return csum;
1969                 offset += copy;
1970                 pos     = copy;
1971         }
1972 
1973         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1974                 int end;
1975                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1976 
1977                 WARN_ON(start > offset + len);
1978 
1979                 end = start + skb_frag_size(frag);
1980                 if ((copy = end - offset) > 0) {
1981                         __wsum csum2;
1982                         u8 *vaddr;
1983 
1984                         if (copy > len)
1985                                 copy = len;
1986                         vaddr = kmap_atomic(skb_frag_page(frag));
1987                         csum2 = ops->update(vaddr + frag->page_offset +
1988                                             offset - start, copy, 0);
1989                         kunmap_atomic(vaddr);
1990                         csum = ops->combine(csum, csum2, pos, copy);
1991                         if (!(len -= copy))
1992                                 return csum;
1993                         offset += copy;
1994                         pos    += copy;
1995                 }
1996                 start = end;
1997         }
1998 
1999         skb_walk_frags(skb, frag_iter) {
2000                 int end;
2001 
2002                 WARN_ON(start > offset + len);
2003 
2004                 end = start + frag_iter->len;
2005                 if ((copy = end - offset) > 0) {
2006                         __wsum csum2;
2007                         if (copy > len)
2008                                 copy = len;
2009                         csum2 = __skb_checksum(frag_iter, offset - start,
2010                                                copy, 0, ops);
2011                         csum = ops->combine(csum, csum2, pos, copy);
2012                         if ((len -= copy) == 0)
2013                                 return csum;
2014                         offset += copy;
2015                         pos    += copy;
2016                 }
2017                 start = end;
2018         }
2019         BUG_ON(len);
2020 
2021         return csum;
2022 }
2023 EXPORT_SYMBOL(__skb_checksum);
2024 
2025 __wsum skb_checksum(const struct sk_buff *skb, int offset,
2026                     int len, __wsum csum)
2027 {
2028         const struct skb_checksum_ops ops = {
2029                 .update  = csum_partial_ext,
2030                 .combine = csum_block_add_ext,
2031         };
2032 
2033         return __skb_checksum(skb, offset, len, csum, &ops);
2034 }
2035 EXPORT_SYMBOL(skb_checksum);
2036 
2037 /* Both of above in one bottle. */
2038 
2039 __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset,
2040                                     u8 *to, int len, __wsum csum)
2041 {
2042         int start = skb_headlen(skb);
2043         int i, copy = start - offset;
2044         struct sk_buff *frag_iter;
2045         int pos = 0;
2046 
2047         /* Copy header. */
2048         if (copy > 0) {
2049                 if (copy > len)
2050                         copy = len;
2051                 csum = csum_partial_copy_nocheck(skb->data + offset, to,
2052                                                  copy, csum);
2053                 if ((len -= copy) == 0)
2054                         return csum;
2055                 offset += copy;
2056                 to     += copy;
2057                 pos     = copy;
2058         }
2059 
2060         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2061                 int end;
2062 
2063                 WARN_ON(start > offset + len);
2064 
2065                 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
2066                 if ((copy = end - offset) > 0) {
2067                         __wsum csum2;
2068                         u8 *vaddr;
2069                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2070 
2071                         if (copy > len)
2072                                 copy = len;
2073                         vaddr = kmap_atomic(skb_frag_page(frag));
2074                         csum2 = csum_partial_copy_nocheck(vaddr +
2075                                                           frag->page_offset +
2076                                                           offset - start, to,
2077                                                           copy, 0);
2078                         kunmap_atomic(vaddr);
2079                         csum = csum_block_add(csum, csum2, pos);
2080                         if (!(len -= copy))
2081                                 return csum;
2082                         offset += copy;
2083                         to     += copy;
2084                         pos    += copy;
2085                 }
2086                 start = end;
2087         }
2088 
2089         skb_walk_frags(skb, frag_iter) {
2090                 __wsum csum2;
2091                 int end;
2092 
2093                 WARN_ON(start > offset + len);
2094 
2095                 end = start + frag_iter->len;
2096                 if ((copy = end - offset) > 0) {
2097                         if (copy > len)
2098                                 copy = len;
2099                         csum2 = skb_copy_and_csum_bits(frag_iter,
2100                                                        offset - start,
2101                                                        to, copy, 0);
2102                         csum = csum_block_add(csum, csum2, pos);
2103                         if ((len -= copy) == 0)
2104                                 return csum;
2105                         offset += copy;
2106                         to     += copy;
2107                         pos    += copy;
2108                 }
2109                 start = end;
2110         }
2111         BUG_ON(len);
2112         return csum;
2113 }
2114 EXPORT_SYMBOL(skb_copy_and_csum_bits);
2115 
2116  /**
2117  *      skb_zerocopy_headlen - Calculate headroom needed for skb_zerocopy()
2118  *      @from: source buffer
2119  *
2120  *      Calculates the amount of linear headroom needed in the 'to' skb passed
2121  *      into skb_zerocopy().
2122  */
2123 unsigned int
2124 skb_zerocopy_headlen(const struct sk_buff *from)
2125 {
2126         unsigned int hlen = 0;
2127 
2128         if (!from->head_frag ||
2129             skb_headlen(from) < L1_CACHE_BYTES ||
2130             skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS)
2131                 hlen = skb_headlen(from);
2132 
2133         if (skb_has_frag_list(from))
2134                 hlen = from->len;
2135 
2136         return hlen;
2137 }
2138 EXPORT_SYMBOL_GPL(skb_zerocopy_headlen);
2139 
2140 /**
2141  *      skb_zerocopy - Zero copy skb to skb
2142  *      @to: destination buffer
2143  *      @from: source buffer
2144  *      @len: number of bytes to copy from source buffer
2145  *      @hlen: size of linear headroom in destination buffer
2146  *
2147  *      Copies up to `len` bytes from `from` to `to` by creating references
2148  *      to the frags in the source buffer.
2149  *
2150  *      The `hlen` as calculated by skb_zerocopy_headlen() specifies the
2151  *      headroom in the `to` buffer.
2152  *
2153  *      Return value:
2154  *      0: everything is OK
2155  *      -ENOMEM: couldn't orphan frags of @from due to lack of memory
2156  *      -EFAULT: skb_copy_bits() found some problem with skb geometry
2157  */
2158 int
2159 skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen)
2160 {
2161         int i, j = 0;
2162         int plen = 0; /* length of skb->head fragment */
2163         int ret;
2164         struct page *page;
2165         unsigned int offset;
2166 
2167         BUG_ON(!from->head_frag && !hlen);
2168 
2169         /* dont bother with small payloads */
2170         if (len <= skb_tailroom(to))
2171                 return skb_copy_bits(from, 0, skb_put(to, len), len);
2172 
2173         if (hlen) {
2174                 ret = skb_copy_bits(from, 0, skb_put(to, hlen), hlen);
2175                 if (unlikely(ret))
2176                         return ret;
2177                 len -= hlen;
2178         } else {
2179                 plen = min_t(int, skb_headlen(from), len);
2180                 if (plen) {
2181                         page = virt_to_head_page(from->head);
2182                         offset = from->data - (unsigned char *)page_address(page);
2183                         __skb_fill_page_desc(to, 0, page, offset, plen);
2184                         get_page(page);
2185                         j = 1;
2186                         len -= plen;
2187                 }
2188         }
2189 
2190         to->truesize += len + plen;
2191         to->len += len + plen;
2192         to->data_len += len + plen;
2193 
2194         if (unlikely(skb_orphan_frags(from, GFP_ATOMIC))) {
2195                 skb_tx_error(from);
2196                 return -ENOMEM;
2197         }
2198 
2199         for (i = 0; i < skb_shinfo(from)->nr_frags; i++) {
2200                 if (!len)
2201                         break;
2202                 skb_shinfo(to)->frags[j] = skb_shinfo(from)->frags[i];
2203                 skb_shinfo(to)->frags[j].size = min_t(int, skb_shinfo(to)->frags[j].size, len);
2204                 len -= skb_shinfo(to)->frags[j].size;
2205                 skb_frag_ref(to, j);
2206                 j++;
2207         }
2208         skb_shinfo(to)->nr_frags = j;
2209 
2210         return 0;
2211 }
2212 EXPORT_SYMBOL_GPL(skb_zerocopy);
2213 
2214 void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to)
2215 {
2216         __wsum csum;
2217         long csstart;
2218 
2219         if (skb->ip_summed == CHECKSUM_PARTIAL)
2220                 csstart = skb_checksum_start_offset(skb);
2221         else
2222                 csstart = skb_headlen(skb);
2223 
2224         BUG_ON(csstart > skb_headlen(skb));
2225 
2226         skb_copy_from_linear_data(skb, to, csstart);
2227 
2228         csum = 0;
2229         if (csstart != skb->len)
2230                 csum = skb_copy_and_csum_bits(skb, csstart, to + csstart,
2231                                               skb->len - csstart, 0);
2232 
2233         if (skb->ip_summed == CHECKSUM_PARTIAL) {
2234                 long csstuff = csstart + skb->csum_offset;
2235 
2236                 *((__sum16 *)(to + csstuff)) = csum_fold(csum);
2237         }
2238 }
2239 EXPORT_SYMBOL(skb_copy_and_csum_dev);
2240 
2241 /**
2242  *      skb_dequeue - remove from the head of the queue
2243  *      @list: list to dequeue from
2244  *
2245  *      Remove the head of the list. The list lock is taken so the function
2246  *      may be used safely with other locking list functions. The head item is
2247  *      returned or %NULL if the list is empty.
2248  */
2249 
2250 struct sk_buff *skb_dequeue(struct sk_buff_head *list)
2251 {
2252         unsigned long flags;
2253         struct sk_buff *result;
2254 
2255         spin_lock_irqsave(&list->lock, flags);
2256         result = __skb_dequeue(list);
2257         spin_unlock_irqrestore(&list->lock, flags);
2258         return result;
2259 }
2260 EXPORT_SYMBOL(skb_dequeue);
2261 
2262 /**
2263  *      skb_dequeue_tail - remove from the tail of the queue
2264  *      @list: list to dequeue from
2265  *
2266  *      Remove the tail of the list. The list lock is taken so the function
2267  *      may be used safely with other locking list functions. The tail item is
2268  *      returned or %NULL if the list is empty.
2269  */
2270 struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list)
2271 {
2272         unsigned long flags;
2273         struct sk_buff *result;
2274 
2275         spin_lock_irqsave(&list->lock, flags);
2276         result = __skb_dequeue_tail(list);
2277         spin_unlock_irqrestore(&list->lock, flags);
2278         return result;
2279 }
2280 EXPORT_SYMBOL(skb_dequeue_tail);
2281 
2282 /**
2283  *      skb_queue_purge - empty a list
2284  *      @list: list to empty
2285  *
2286  *      Delete all buffers on an &sk_buff list. Each buffer is removed from
2287  *      the list and one reference dropped. This function takes the list
2288  *      lock and is atomic with respect to other list locking functions.
2289  */
2290 void skb_queue_purge(struct sk_buff_head *list)
2291 {
2292         struct sk_buff *skb;
2293         while ((skb = skb_dequeue(list)) != NULL)
2294                 kfree_skb(skb);
2295 }
2296 EXPORT_SYMBOL(skb_queue_purge);
2297 
2298 /**
2299  *      skb_queue_head - queue a buffer at the list head
2300  *      @list: list to use
2301  *      @newsk: buffer to queue
2302  *
2303  *      Queue a buffer at the start of the list. This function takes the
2304  *      list lock and can be used safely with other locking &sk_buff functions
2305  *      safely.
2306  *
2307  *      A buffer cannot be placed on two lists at the same time.
2308  */
2309 void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk)
2310 {
2311         unsigned long flags;
2312 
2313         spin_lock_irqsave(&list->lock, flags);
2314         __skb_queue_head(list, newsk);
2315         spin_unlock_irqrestore(&list->lock, flags);
2316 }
2317 EXPORT_SYMBOL(skb_queue_head);
2318 
2319 /**
2320  *      skb_queue_tail - queue a buffer at the list tail
2321  *      @list: list to use
2322  *      @newsk: buffer to queue
2323  *
2324  *      Queue a buffer at the tail of the list. This function takes the
2325  *      list lock and can be used safely with other locking &sk_buff functions
2326  *      safely.
2327  *
2328  *      A buffer cannot be placed on two lists at the same time.
2329  */
2330 void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk)
2331 {
2332         unsigned long flags;
2333 
2334         spin_lock_irqsave(&list->lock, flags);
2335         __skb_queue_tail(list, newsk);
2336         spin_unlock_irqrestore(&list->lock, flags);
2337 }
2338 EXPORT_SYMBOL(skb_queue_tail);
2339 
2340 /**
2341  *      skb_unlink      -       remove a buffer from a list
2342  *      @skb: buffer to remove
2343  *      @list: list to use
2344  *
2345  *      Remove a packet from a list. The list locks are taken and this
2346  *      function is atomic with respect to other list locked calls
2347  *
2348  *      You must know what list the SKB is on.
2349  */
2350 void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
2351 {
2352         unsigned long flags;
2353 
2354         spin_lock_irqsave(&list->lock, flags);
2355         __skb_unlink(skb, list);
2356         spin_unlock_irqrestore(&list->lock, flags);
2357 }
2358 EXPORT_SYMBOL(skb_unlink);
2359 
2360 /**
2361  *      skb_append      -       append a buffer
2362  *      @old: buffer to insert after
2363  *      @newsk: buffer to insert
2364  *      @list: list to use
2365  *
2366  *      Place a packet after a given packet in a list. The list locks are taken
2367  *      and this function is atomic with respect to other list locked calls.
2368  *      A buffer cannot be placed on two lists at the same time.
2369  */
2370 void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
2371 {
2372         unsigned long flags;
2373 
2374         spin_lock_irqsave(&list->lock, flags);
2375         __skb_queue_after(list, old, newsk);
2376         spin_unlock_irqrestore(&list->lock, flags);
2377 }
2378 EXPORT_SYMBOL(skb_append);
2379 
2380 /**
2381  *      skb_insert      -       insert a buffer
2382  *      @old: buffer to insert before
2383  *      @newsk: buffer to insert
2384  *      @list: list to use
2385  *
2386  *      Place a packet before a given packet in a list. The list locks are
2387  *      taken and this function is atomic with respect to other list locked
2388  *      calls.
2389  *
2390  *      A buffer cannot be placed on two lists at the same time.
2391  */
2392 void skb_insert(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
2393 {
2394         unsigned long flags;
2395 
2396         spin_lock_irqsave(&list->lock, flags);
2397         __skb_insert(newsk, old->prev, old, list);
2398         spin_unlock_irqrestore(&list->lock, flags);
2399 }
2400 EXPORT_SYMBOL(skb_insert);
2401 
2402 static inline void skb_split_inside_header(struct sk_buff *skb,
2403                                            struct sk_buff* skb1,
2404                                            const u32 len, const int pos)
2405 {
2406         int i;
2407 
2408         skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len),
2409                                          pos - len);
2410         /* And move data appendix as is. */
2411         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
2412                 skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i];
2413 
2414         skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags;
2415         skb_shinfo(skb)->nr_frags  = 0;
2416         skb1->data_len             = skb->data_len;
2417         skb1->len                  += skb1->data_len;
2418         skb->data_len              = 0;
2419         skb->len                   = len;
2420         skb_set_tail_pointer(skb, len);
2421 }
2422 
2423 static inline void skb_split_no_header(struct sk_buff *skb,
2424                                        struct sk_buff* skb1,
2425                                        const u32 len, int pos)
2426 {
2427         int i, k = 0;
2428         const int nfrags = skb_shinfo(skb)->nr_frags;
2429 
2430         skb_shinfo(skb)->nr_frags = 0;
2431         skb1->len                 = skb1->data_len = skb->len - len;
2432         skb->len                  = len;
2433         skb->data_len             = len - pos;
2434 
2435         for (i = 0; i < nfrags; i++) {
2436                 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
2437 
2438                 if (pos + size > len) {
2439                         skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i];
2440 
2441                         if (pos < len) {
2442                                 /* Split frag.
2443                                  * We have two variants in this case:
2444                                  * 1. Move all the frag to the second
2445                                  *    part, if it is possible. F.e.
2446                                  *    this approach is mandatory for TUX,
2447                                  *    where splitting is expensive.
2448                                  * 2. Split is accurately. We make this.
2449                                  */
2450                                 skb_frag_ref(skb, i);
2451                                 skb_shinfo(skb1)->frags[0].page_offset += len - pos;
2452                                 skb_frag_size_sub(&skb_shinfo(skb1)->frags[0], len - pos);
2453                                 skb_frag_size_set(&skb_shinfo(skb)->frags[i], len - pos);
2454                                 skb_shinfo(skb)->nr_frags++;
2455                         }
2456                         k++;
2457                 } else
2458                         skb_shinfo(skb)->nr_frags++;
2459                 pos += size;
2460         }
2461         skb_shinfo(skb1)->nr_frags = k;
2462 }
2463 
2464 /**
2465  * skb_split - Split fragmented skb to two parts at length len.
2466  * @skb: the buffer to split
2467  * @skb1: the buffer to receive the second part
2468  * @len: new length for skb
2469  */
2470 void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len)
2471 {
2472         int pos = skb_headlen(skb);
2473 
2474         skb_shinfo(skb1)->tx_flags = skb_shinfo(skb)->tx_flags & SKBTX_SHARED_FRAG;
2475         if (len < pos)  /* Split line is inside header. */
2476                 skb_split_inside_header(skb, skb1, len, pos);
2477         else            /* Second chunk has no header, nothing to copy. */
2478                 skb_split_no_header(skb, skb1, len, pos);
2479 }
2480 EXPORT_SYMBOL(skb_split);
2481 
2482 /* Shifting from/to a cloned skb is a no-go.
2483  *
2484  * Caller cannot keep skb_shinfo related pointers past calling here!
2485  */
2486 static int skb_prepare_for_shift(struct sk_buff *skb)
2487 {
2488         return skb_cloned(skb) && pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2489 }
2490 
2491 /**
2492  * skb_shift - Shifts paged data partially from skb to another
2493  * @tgt: buffer into which tail data gets added
2494  * @skb: buffer from which the paged data comes from
2495  * @shiftlen: shift up to this many bytes
2496  *
2497  * Attempts to shift up to shiftlen worth of bytes, which may be less than
2498  * the length of the skb, from skb to tgt. Returns number bytes shifted.
2499  * It's up to caller to free skb if everything was shifted.
2500  *
2501  * If @tgt runs out of frags, the whole operation is aborted.
2502  *
2503  * Skb cannot include anything else but paged data while tgt is allowed
2504  * to have non-paged data as well.
2505  *
2506  * TODO: full sized shift could be optimized but that would need
2507  * specialized skb free'er to handle frags without up-to-date nr_frags.
2508  */
2509 int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen)
2510 {
2511         int from, to, merge, todo;
2512         struct skb_frag_struct *fragfrom, *fragto;
2513 
2514         BUG_ON(shiftlen > skb->len);
2515         BUG_ON(skb_headlen(skb));       /* Would corrupt stream */
2516 
2517         todo = shiftlen;
2518         from = 0;
2519         to = skb_shinfo(tgt)->nr_frags;
2520         fragfrom = &skb_shinfo(skb)->frags[from];
2521 
2522         /* Actual merge is delayed until the point when we know we can
2523          * commit all, so that we don't have to undo partial changes
2524          */
2525         if (!to ||
2526             !skb_can_coalesce(tgt, to, skb_frag_page(fragfrom),
2527                               fragfrom->page_offset)) {
2528                 merge = -1;
2529         } else {
2530                 merge = to - 1;
2531 
2532                 todo -= skb_frag_size(fragfrom);
2533                 if (todo < 0) {
2534                         if (skb_prepare_for_shift(skb) ||
2535                             skb_prepare_for_shift(tgt))
2536                                 return 0;
2537 
2538                         /* All previous frag pointers might be stale! */
2539                         fragfrom = &skb_shinfo(skb)->frags[from];
2540                         fragto = &skb_shinfo(tgt)->frags[merge];
2541 
2542                         skb_frag_size_add(fragto, shiftlen);
2543                         skb_frag_size_sub(fragfrom, shiftlen);
2544                         fragfrom->page_offset += shiftlen;
2545 
2546                         goto onlymerged;
2547                 }
2548 
2549                 from++;
2550         }
2551 
2552         /* Skip full, not-fitting skb to avoid expensive operations */
2553         if ((shiftlen == skb->len) &&
2554             (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to))
2555                 return 0;
2556 
2557         if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt))
2558                 return 0;
2559 
2560         while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) {
2561                 if (to == MAX_SKB_FRAGS)
2562                         return 0;
2563 
2564                 fragfrom = &skb_shinfo(skb)->frags[from];
2565                 fragto = &skb_shinfo(tgt)->frags[to];
2566 
2567                 if (todo >= skb_frag_size(fragfrom)) {
2568                         *fragto = *fragfrom;
2569                         todo -= skb_frag_size(fragfrom);
2570                         from++;
2571                         to++;
2572 
2573                 } else {
2574                         __skb_frag_ref(fragfrom);
2575                         fragto->page = fragfrom->page;
2576                         fragto->page_offset = fragfrom->page_offset;
2577                         skb_frag_size_set(fragto, todo);
2578 
2579                         fragfrom->page_offset += todo;
2580                         skb_frag_size_sub(fragfrom, todo);
2581                         todo = 0;
2582 
2583                         to++;
2584                         break;
2585                 }
2586         }
2587 
2588         /* Ready to "commit" this state change to tgt */
2589         skb_shinfo(tgt)->nr_frags = to;
2590 
2591         if (merge >= 0) {
2592                 fragfrom = &skb_shinfo(skb)->frags[0];
2593                 fragto = &skb_shinfo(tgt)->frags[merge];
2594 
2595                 skb_frag_size_add(fragto, skb_frag_size(fragfrom));
2596                 __skb_frag_unref(fragfrom);
2597         }
2598 
2599         /* Reposition in the original skb */
2600         to = 0;
2601         while (from < skb_shinfo(skb)->nr_frags)
2602                 skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++];
2603         skb_shinfo(skb)->nr_frags = to;
2604 
2605         BUG_ON(todo > 0 && !skb_shinfo(skb)->nr_frags);
2606 
2607 onlymerged:
2608         /* Most likely the tgt won't ever need its checksum anymore, skb on
2609          * the other hand might need it if it needs to be resent
2610          */
2611         tgt->ip_summed = CHECKSUM_PARTIAL;
2612         skb->ip_summed = CHECKSUM_PARTIAL;
2613 
2614         /* Yak, is it really working this way? Some helper please? */
2615         skb->len -= shiftlen;
2616         skb->data_len -= shiftlen;
2617         skb->truesize -= shiftlen;
2618         tgt->len += shiftlen;
2619         tgt->data_len += shiftlen;
2620         tgt->truesize += shiftlen;
2621 
2622         return shiftlen;
2623 }
2624 
2625 /**
2626  * skb_prepare_seq_read - Prepare a sequential read of skb data
2627  * @skb: the buffer to read
2628  * @from: lower offset of data to be read
2629  * @to: upper offset of data to be read
2630  * @st: state variable
2631  *
2632  * Initializes the specified state variable. Must be called before
2633  * invoking skb_seq_read() for the first time.
2634  */
2635 void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
2636                           unsigned int to, struct skb_seq_state *st)
2637 {
2638         st->lower_offset = from;
2639         st->upper_offset = to;
2640         st->root_skb = st->cur_skb = skb;
2641         st->frag_idx = st->stepped_offset = 0;
2642         st->frag_data = NULL;
2643 }
2644 EXPORT_SYMBOL(skb_prepare_seq_read);
2645 
2646 /**
2647  * skb_seq_read - Sequentially read skb data
2648  * @consumed: number of bytes consumed by the caller so far
2649  * @data: destination pointer for data to be returned
2650  * @st: state variable
2651  *
2652  * Reads a block of skb data at @consumed relative to the
2653  * lower offset specified to skb_prepare_seq_read(). Assigns
2654  * the head of the data block to @data and returns the length
2655  * of the block or 0 if the end of the skb data or the upper
2656  * offset has been reached.
2657  *
2658  * The caller is not required to consume all of the data
2659  * returned, i.e. @consumed is typically set to the number
2660  * of bytes already consumed and the next call to
2661  * skb_seq_read() will return the remaining part of the block.
2662  *
2663  * Note 1: The size of each block of data returned can be arbitrary,
2664  *       this limitation is the cost for zerocopy sequential
2665  *       reads of potentially non linear data.
2666  *
2667  * Note 2: Fragment lists within fragments are not implemented
2668  *       at the moment, state->root_skb could be replaced with
2669  *       a stack for this purpose.
2670  */
2671 unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
2672                           struct skb_seq_state *st)
2673 {
2674         unsigned int block_limit, abs_offset = consumed + st->lower_offset;
2675         skb_frag_t *frag;
2676 
2677         if (unlikely(abs_offset >= st->upper_offset)) {
2678                 if (st->frag_data) {
2679                         kunmap_atomic(st->frag_data);
2680                         st->frag_data = NULL;
2681                 }
2682                 return 0;
2683         }
2684 
2685 next_skb:
2686         block_limit = skb_headlen(st->cur_skb) + st->stepped_offset;
2687 
2688         if (abs_offset < block_limit && !st->frag_data) {
2689                 *data = st->cur_skb->data + (abs_offset - st->stepped_offset);
2690                 return block_limit - abs_offset;
2691         }
2692 
2693         if (st->frag_idx == 0 && !st->frag_data)
2694                 st->stepped_offset += skb_headlen(st->cur_skb);
2695 
2696         while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) {
2697                 frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx];
2698                 block_limit = skb_frag_size(frag) + st->stepped_offset;
2699 
2700                 if (abs_offset < block_limit) {
2701                         if (!st->frag_data)
2702                                 st->frag_data = kmap_atomic(skb_frag_page(frag));
2703 
2704                         *data = (u8 *) st->frag_data + frag->page_offset +
2705                                 (abs_offset - st->stepped_offset);
2706 
2707                         return block_limit - abs_offset;
2708                 }
2709 
2710                 if (st->frag_data) {
2711                         kunmap_atomic(st->frag_data);
2712                         st->frag_data = NULL;
2713                 }
2714 
2715                 st->frag_idx++;
2716                 st->stepped_offset += skb_frag_size(frag);
2717         }
2718 
2719         if (st->frag_data) {
2720                 kunmap_atomic(st->frag_data);
2721                 st->frag_data = NULL;
2722         }
2723 
2724         if (st->root_skb == st->cur_skb && skb_has_frag_list(st->root_skb)) {
2725                 st->cur_skb = skb_shinfo(st->root_skb)->frag_list;
2726                 st->frag_idx = 0;
2727                 goto next_skb;
2728         } else if (st->cur_skb->next) {
2729                 st->cur_skb = st->cur_skb->next;
2730                 st->frag_idx = 0;
2731                 goto next_skb;
2732         }
2733 
2734         return 0;
2735 }
2736 EXPORT_SYMBOL(skb_seq_read);
2737 
2738 /**
2739  * skb_abort_seq_read - Abort a sequential read of skb data
2740  * @st: state variable
2741  *
2742  * Must be called if skb_seq_read() was not called until it
2743  * returned 0.
2744  */
2745 void skb_abort_seq_read(struct skb_seq_state *st)
2746 {
2747         if (st->frag_data)
2748                 kunmap_atomic(st->frag_data);
2749 }
2750 EXPORT_SYMBOL(skb_abort_seq_read);
2751 
2752 #define TS_SKB_CB(state)        ((struct skb_seq_state *) &((state)->cb))
2753 
2754 static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text,
2755                                           struct ts_config *conf,
2756                                           struct ts_state *state)
2757 {
2758         return skb_seq_read(offset, text, TS_SKB_CB(state));
2759 }
2760 
2761 static void skb_ts_finish(struct ts_config *conf, struct ts_state *state)
2762 {
2763         skb_abort_seq_read(TS_SKB_CB(state));
2764 }
2765 
2766 /**
2767  * skb_find_text - Find a text pattern in skb data
2768  * @skb: the buffer to look in
2769  * @from: search offset
2770  * @to: search limit
2771  * @config: textsearch configuration
2772  * @state: uninitialized textsearch state variable
2773  *
2774  * Finds a pattern in the skb data according to the specified
2775  * textsearch configuration. Use textsearch_next() to retrieve
2776  * subsequent occurrences of the pattern. Returns the offset
2777  * to the first occurrence or UINT_MAX if no match was found.
2778  */
2779 unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
2780                            unsigned int to, struct ts_config *config,
2781                            struct ts_state *state)
2782 {
2783         unsigned int ret;
2784 
2785         config->get_next_block = skb_ts_get_next_block;
2786         config->finish = skb_ts_finish;
2787 
2788         skb_prepare_seq_read(skb, from, to, TS_SKB_CB(state));
2789 
2790         ret = textsearch_find(config, state);
2791         return (ret <= to - from ? ret : UINT_MAX);
2792 }
2793 EXPORT_SYMBOL(skb_find_text);
2794 
2795 /**
2796  * skb_append_datato_frags - append the user data to a skb
2797  * @sk: sock  structure
2798  * @skb: skb structure to be appended with user data.
2799  * @getfrag: call back function to be used for getting the user data
2800  * @from: pointer to user message iov
2801  * @length: length of the iov message
2802  *
2803  * Description: This procedure append the user data in the fragment part
2804  * of the skb if any page alloc fails user this procedure returns  -ENOMEM
2805  */
2806 int skb_append_datato_frags(struct sock *sk, struct sk_buff *skb,
2807                         int (*getfrag)(void *from, char *to, int offset,
2808                                         int len, int odd, struct sk_buff *skb),
2809                         void *from, int length)
2810 {
2811         int frg_cnt = skb_shinfo(skb)->nr_frags;
2812         int copy;
2813         int offset = 0;
2814         int ret;
2815         struct page_frag *pfrag = &current->task_frag;
2816 
2817         do {
2818                 /* Return error if we don't have space for new frag */
2819                 if (frg_cnt >= MAX_SKB_FRAGS)
2820                         return -EMSGSIZE;
2821 
2822                 if (!sk_page_frag_refill(sk, pfrag))
2823                         return -ENOMEM;
2824 
2825                 /* copy the user data to page */
2826                 copy = min_t(int, length, pfrag->size - pfrag->offset);
2827 
2828                 ret = getfrag(from, page_address(pfrag->page) + pfrag->offset,
2829                               offset, copy, 0, skb);
2830                 if (ret < 0)
2831                         return -EFAULT;
2832 
2833                 /* copy was successful so update the size parameters */
2834                 skb_fill_page_desc(skb, frg_cnt, pfrag->page, pfrag->offset,
2835                                    copy);
2836                 frg_cnt++;
2837                 pfrag->offset += copy;
2838                 get_page(pfrag->page);
2839 
2840                 skb->truesize += copy;
2841                 atomic_add(copy, &sk->sk_wmem_alloc);
2842                 skb->len += copy;
2843                 skb->data_len += copy;
2844                 offset += copy;
2845                 length -= copy;
2846 
2847         } while (length > 0);
2848 
2849         return 0;
2850 }
2851 EXPORT_SYMBOL(skb_append_datato_frags);
2852 
2853 /**
2854  *      skb_pull_rcsum - pull skb and update receive checksum
2855  *      @skb: buffer to update
2856  *      @len: length of data pulled
2857  *
2858  *      This function performs an skb_pull on the packet and updates
2859  *      the CHECKSUM_COMPLETE checksum.  It should be used on
2860  *      receive path processing instead of skb_pull unless you know
2861  *      that the checksum difference is zero (e.g., a valid IP header)
2862  *      or you are setting ip_summed to CHECKSUM_NONE.
2863  */
2864 unsigned char *skb_pull_rcsum(struct sk_buff *skb, unsigned int len)
2865 {
2866         BUG_ON(len > skb->len);
2867         skb->len -= len;
2868         BUG_ON(skb->len < skb->data_len);
2869         skb_postpull_rcsum(skb, skb->data, len);
2870         return skb->data += len;
2871 }
2872 EXPORT_SYMBOL_GPL(skb_pull_rcsum);
2873 
2874 /**
2875  *      skb_segment - Perform protocol segmentation on skb.
2876  *      @head_skb: buffer to segment
2877  *      @features: features for the output path (see dev->features)
2878  *
2879  *      This function performs segmentation on the given skb.  It returns
2880  *      a pointer to the first in a list of new skbs for the segments.
2881  *      In case of error it returns ERR_PTR(err).
2882  */
2883 struct sk_buff *skb_segment(struct sk_buff *head_skb,
2884                             netdev_features_t features)
2885 {
2886         struct sk_buff *segs = NULL;
2887         struct sk_buff *tail = NULL;
2888         struct sk_buff *list_skb = skb_shinfo(head_skb)->frag_list;
2889         skb_frag_t *frag = skb_shinfo(head_skb)->frags;
2890         unsigned int mss = skb_shinfo(head_skb)->gso_size;
2891         unsigned int doffset = head_skb->data - skb_mac_header(head_skb);
2892         struct sk_buff *frag_skb = head_skb;
2893         unsigned int offset = doffset;
2894         unsigned int tnl_hlen = skb_tnl_header_len(head_skb);
2895         unsigned int headroom;
2896         unsigned int len;
2897         __be16 proto;
2898         bool csum;
2899         int sg = !!(features & NETIF_F_SG);
2900         int nfrags = skb_shinfo(head_skb)->nr_frags;
2901         int err = -ENOMEM;
2902         int i = 0;
2903         int pos;
2904         int dummy;
2905 
2906         __skb_push(head_skb, doffset);
2907         proto = skb_network_protocol(head_skb, &dummy);
2908         if (unlikely(!proto))
2909                 return ERR_PTR(-EINVAL);
2910 
2911         csum = !head_skb->encap_hdr_csum &&
2912             !!can_checksum_protocol(features, proto);
2913 
2914         headroom = skb_headroom(head_skb);
2915         pos = skb_headlen(head_skb);
2916 
2917         do {
2918                 struct sk_buff *nskb;
2919                 skb_frag_t *nskb_frag;
2920                 int hsize;
2921                 int size;
2922 
2923                 len = head_skb->len - offset;
2924                 if (len > mss)
2925                         len = mss;
2926 
2927                 hsize = skb_headlen(head_skb) - offset;
2928                 if (hsize < 0)
2929                         hsize = 0;
2930                 if (hsize > len || !sg)
2931                         hsize = len;
2932 
2933                 if (!hsize && i >= nfrags && skb_headlen(list_skb) &&
2934                     (skb_headlen(list_skb) == len || sg)) {
2935                         BUG_ON(skb_headlen(list_skb) > len);
2936 
2937                         i = 0;
2938                         nfrags = skb_shinfo(list_skb)->nr_frags;
2939                         frag = skb_shinfo(list_skb)->frags;
2940                         frag_skb = list_skb;
2941                         pos += skb_headlen(list_skb);
2942 
2943                         while (pos < offset + len) {
2944                                 BUG_ON(i >= nfrags);
2945 
2946                                 size = skb_frag_size(frag);
2947                                 if (pos + size > offset + len)
2948                                         break;
2949 
2950                                 i++;
2951                                 pos += size;
2952                                 frag++;
2953                         }
2954 
2955                         nskb = skb_clone(list_skb, GFP_ATOMIC);
2956                         list_skb = list_skb->next;
2957 
2958                         if (unlikely(!nskb))
2959                                 goto err;
2960 
2961                         if (unlikely(pskb_trim(nskb, len))) {
2962                                 kfree_skb(nskb);
2963                                 goto err;
2964                         }
2965 
2966                         hsize = skb_end_offset(nskb);
2967                         if (skb_cow_head(nskb, doffset + headroom)) {
2968                                 kfree_skb(nskb);
2969                                 goto err;
2970                         }
2971 
2972                         nskb->truesize += skb_end_offset(nskb) - hsize;
2973                         skb_release_head_state(nskb);
2974                         __skb_push(nskb, doffset);
2975                 } else {
2976                         nskb = __alloc_skb(hsize + doffset + headroom,
2977                                            GFP_ATOMIC, skb_alloc_rx_flag(head_skb),
2978                                            NUMA_NO_NODE);
2979 
2980                         if (unlikely(!nskb))
2981                                 goto err;
2982 
2983                         skb_reserve(nskb, headroom);
2984                         __skb_put(nskb, doffset);
2985                 }
2986 
2987                 if (segs)
2988                         tail->next = nskb;
2989                 else
2990                         segs = nskb;
2991                 tail = nskb;
2992 
2993                 __copy_skb_header(nskb, head_skb);
2994 
2995                 skb_headers_offset_update(nskb, skb_headroom(nskb) - headroom);
2996                 skb_reset_mac_len(nskb);
2997 
2998                 skb_copy_from_linear_data_offset(head_skb, -tnl_hlen,
2999                                                  nskb->data - tnl_hlen,
3000                                                  doffset + tnl_hlen);
3001 
3002                 if (nskb->len == len + doffset)
3003                         goto perform_csum_check;
3004 
3005                 if (!sg) {
3006                         nskb->ip_summed = CHECKSUM_NONE;
3007                         nskb->csum = skb_copy_and_csum_bits(head_skb, offset,
3008                                                             skb_put(nskb, len),
3009                                                             len, 0);
3010                         SKB_GSO_CB(nskb)->csum_start =
3011                             skb_headroom(nskb) + doffset;
3012                         continue;
3013                 }
3014 
3015                 nskb_frag = skb_shinfo(nskb)->frags;
3016 
3017                 skb_copy_from_linear_data_offset(head_skb, offset,
3018                                                  skb_put(nskb, hsize), hsize);
3019 
3020                 skb_shinfo(nskb)->tx_flags = skb_shinfo(head_skb)->tx_flags &
3021                         SKBTX_SHARED_FRAG;
3022 
3023                 while (pos < offset + len) {
3024                         if (i >= nfrags) {
3025                                 BUG_ON(skb_headlen(list_skb));
3026 
3027                                 i = 0;
3028                                 nfrags = skb_shinfo(list_skb)->nr_frags;
3029                                 frag = skb_shinfo(list_skb)->frags;
3030                                 frag_skb = list_skb;
3031 
3032                                 BUG_ON(!nfrags);
3033 
3034                                 list_skb = list_skb->next;
3035                         }
3036 
3037                         if (unlikely(skb_shinfo(nskb)->nr_frags >=
3038                                      MAX_SKB_FRAGS)) {
3039                                 net_warn_ratelimited(
3040                                         "skb_segment: too many frags: %u %u\n",
3041                                         pos, mss);
3042                                 goto err;
3043                         }
3044 
3045                         if (unlikely(skb_orphan_frags(frag_skb, GFP_ATOMIC)))
3046                                 goto err;
3047 
3048                         *nskb_frag = *frag;
3049                         __skb_frag_ref(nskb_frag);
3050                         size = skb_frag_size(nskb_frag);
3051 
3052                         if (pos < offset) {
3053                                 nskb_frag->page_offset += offset - pos;
3054                                 skb_frag_size_sub(nskb_frag, offset - pos);
3055                         }
3056 
3057                         skb_shinfo(nskb)->nr_frags++;
3058 
3059                         if (pos + size <= offset + len) {
3060                                 i++;
3061                                 frag++;
3062                                 pos += size;
3063                         } else {
3064                                 skb_frag_size_sub(nskb_frag, pos + size - (offset + len));
3065                                 goto skip_fraglist;
3066                         }
3067 
3068                         nskb_frag++;
3069                 }
3070 
3071 skip_fraglist:
3072                 nskb->data_len = len - hsize;
3073                 nskb->len += nskb->data_len;
3074                 nskb->truesize += nskb->data_len;
3075 
3076 perform_csum_check:
3077                 if (!csum) {
3078                         nskb->csum = skb_checksum(nskb, doffset,
3079                                                   nskb->len - doffset, 0);
3080                         nskb->ip_summed = CHECKSUM_NONE;
3081                         SKB_GSO_CB(nskb)->csum_start =
3082                             skb_headroom(nskb) + doffset;
3083                 }
3084         } while ((offset += len) < head_skb->len);
3085 
3086         /* Some callers want to get the end of the list.
3087          * Put it in segs->prev to avoid walking the list.
3088          * (see validate_xmit_skb_list() for example)
3089          */
3090         segs->prev = tail;
3091         return segs;
3092 
3093 err:
3094         kfree_skb_list(segs);
3095         return ERR_PTR(err);
3096 }
3097 EXPORT_SYMBOL_GPL(skb_segment);
3098 
3099 int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb)
3100 {
3101         struct skb_shared_info *pinfo, *skbinfo = skb_shinfo(skb);
3102         unsigned int offset = skb_gro_offset(skb);
3103         unsigned int headlen = skb_headlen(skb);
3104         struct sk_buff *nskb, *lp, *p = *head;
3105         unsigned int len = skb_gro_len(skb);
3106         unsigned int delta_truesize;
3107         unsigned int headroom;
3108 
3109         if (unlikely(p->len + len >= 65536))
3110                 return -E2BIG;
3111 
3112         lp = NAPI_GRO_CB(p)->last;
3113         pinfo = skb_shinfo(lp);
3114 
3115         if (headlen <= offset) {
3116                 skb_frag_t *frag;
3117                 skb_frag_t *frag2;
3118                 int i = skbinfo->nr_frags;
3119                 int nr_frags = pinfo->nr_frags + i;
3120 
3121                 if (nr_frags > MAX_SKB_FRAGS)
3122                         goto merge;
3123 
3124                 offset -= headlen;
3125                 pinfo->nr_frags = nr_frags;
3126                 skbinfo->nr_frags = 0;
3127 
3128                 frag = pinfo->frags + nr_frags;
3129                 frag2 = skbinfo->frags + i;
3130                 do {
3131                         *--frag = *--frag2;
3132                 } while (--i);
3133 
3134                 frag->page_offset += offset;
3135                 skb_frag_size_sub(frag, offset);
3136 
3137                 /* all fragments truesize : remove (head size + sk_buff) */
3138                 delta_truesize = skb->truesize -
3139                                  SKB_TRUESIZE(skb_end_offset(skb));
3140 
3141                 skb->truesize -= skb->data_len;
3142                 skb->len -= skb->data_len;
3143                 skb->data_len = 0;
3144 
3145                 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE;
3146                 goto done;
3147         } else if (skb->head_frag) {
3148                 int nr_frags = pinfo->nr_frags;
3149                 skb_frag_t *frag = pinfo->frags + nr_frags;
3150                 struct page *page = virt_to_head_page(skb->head);
3151                 unsigned int first_size = headlen - offset;
3152                 unsigned int first_offset;
3153 
3154                 if (nr_frags + 1 + skbinfo->nr_frags > MAX_SKB_FRAGS)
3155                         goto merge;
3156 
3157                 first_offset = skb->data -
3158                                (unsigned char *)page_address(page) +
3159                                offset;
3160 
3161                 pinfo->nr_frags = nr_frags + 1 + skbinfo->nr_frags;
3162 
3163                 frag->page.p      = page;
3164                 frag->page_offset = first_offset;
3165                 skb_frag_size_set(frag, first_size);
3166 
3167                 memcpy(frag + 1, skbinfo->frags, sizeof(*frag) * skbinfo->nr_frags);
3168                 /* We dont need to clear skbinfo->nr_frags here */
3169 
3170                 delta_truesize = skb->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff));
3171                 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE_STOLEN_HEAD;
3172                 goto done;
3173         }
3174         /* switch back to head shinfo */
3175         pinfo = skb_shinfo(p);
3176 
3177         if (pinfo->frag_list)
3178                 goto merge;
3179         if (skb_gro_len(p) != pinfo->gso_size)
3180                 return -E2BIG;
3181 
3182         headroom = skb_headroom(p);
3183         nskb = alloc_skb(headroom + skb_gro_offset(p), GFP_ATOMIC);
3184         if (unlikely(!nskb))
3185                 return -ENOMEM;
3186 
3187         __copy_skb_header(nskb, p);
3188         nskb->mac_len = p->mac_len;
3189 
3190         skb_reserve(nskb, headroom);
3191         __skb_put(nskb, skb_gro_offset(p));
3192 
3193         skb_set_mac_header(nskb, skb_mac_header(p) - p->data);
3194         skb_set_network_header(nskb, skb_network_offset(p));
3195         skb_set_transport_header(nskb, skb_transport_offset(p));
3196 
3197         __skb_pull(p, skb_gro_offset(p));
3198         memcpy(skb_mac_header(nskb), skb_mac_header(p),
3199                p->data - skb_mac_header(p));
3200 
3201         skb_shinfo(nskb)->frag_list = p;
3202         skb_shinfo(nskb)->gso_size = pinfo->gso_size;
3203         pinfo->gso_size = 0;
3204         __skb_header_release(p);
3205         NAPI_GRO_CB(nskb)->last = p;
3206 
3207         nskb->data_len += p->len;
3208         nskb->truesize += p->truesize;
3209         nskb->len += p->len;
3210 
3211         *head = nskb;
3212         nskb->next = p->next;
3213         p->next = NULL;
3214 
3215         p = nskb;
3216 
3217 merge:
3218         delta_truesize = skb->truesize;
3219         if (offset > headlen) {
3220                 unsigned int eat = offset - headlen;
3221 
3222                 skbinfo->frags[0].page_offset += eat;
3223                 skb_frag_size_sub(&skbinfo->frags[0], eat);
3224                 skb->data_len -= eat;
3225                 skb->len -= eat;
3226                 offset = headlen;
3227         }
3228 
3229         __skb_pull(skb, offset);
3230 
3231         if (NAPI_GRO_CB(p)->last == p)
3232                 skb_shinfo(p)->frag_list = skb;
3233         else
3234                 NAPI_GRO_CB(p)->last->next = skb;
3235         NAPI_GRO_CB(p)->last = skb;
3236         __skb_header_release(skb);
3237         lp = p;
3238 
3239 done:
3240         NAPI_GRO_CB(p)->count++;
3241         p->data_len += len;
3242         p->truesize += delta_truesize;
3243         p->len += len;
3244         if (lp != p) {
3245                 lp->data_len += len;
3246                 lp->truesize += delta_truesize;
3247                 lp->len += len;
3248         }
3249         NAPI_GRO_CB(skb)->same_flow = 1;
3250         return 0;
3251 }
3252 
3253 void __init skb_init(void)
3254 {
3255         skbuff_head_cache = kmem_cache_create("skbuff_head_cache",
3256                                               sizeof(struct sk_buff),
3257                                               0,
3258                                               SLAB_HWCACHE_ALIGN|SLAB_PANIC,
3259                                               NULL);
3260         skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache",
3261                                                 sizeof(struct sk_buff_fclones),
3262                                                 0,
3263                                                 SLAB_HWCACHE_ALIGN|SLAB_PANIC,
3264                                                 NULL);
3265 }
3266 
3267 /**
3268  *      skb_to_sgvec - Fill a scatter-gather list from a socket buffer
3269  *      @skb: Socket buffer containing the buffers to be mapped
3270  *      @sg: The scatter-gather list to map into
3271  *      @offset: The offset into the buffer's contents to start mapping
3272  *      @len: Length of buffer space to be mapped
3273  *
3274  *      Fill the specified scatter-gather list with mappings/pointers into a
3275  *      region of the buffer space attached to a socket buffer.
3276  */
3277 static int
3278 __skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len)
3279 {
3280         int start = skb_headlen(skb);
3281         int i, copy = start - offset;
3282         struct sk_buff *frag_iter;
3283         int elt = 0;
3284 
3285         if (copy > 0) {
3286                 if (copy > len)
3287                         copy = len;
3288                 sg_set_buf(sg, skb->data + offset, copy);
3289                 elt++;
3290                 if ((len -= copy) == 0)
3291                         return elt;
3292                 offset += copy;
3293         }
3294 
3295         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3296                 int end;
3297 
3298                 WARN_ON(start > offset + len);
3299 
3300                 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
3301                 if ((copy = end - offset) > 0) {
3302                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3303 
3304                         if (copy > len)
3305                                 copy = len;
3306                         sg_set_page(&sg[elt], skb_frag_page(frag), copy,
3307                                         frag->page_offset+offset-start);
3308                         elt++;
3309                         if (!(len -= copy))
3310                                 return elt;
3311                         offset += copy;
3312                 }
3313                 start = end;
3314         }
3315 
3316         skb_walk_frags(skb, frag_iter) {
3317                 int end;
3318 
3319                 WARN_ON(start > offset + len);
3320 
3321                 end = start + frag_iter->len;
3322                 if ((copy = end - offset) > 0) {
3323                         if (copy > len)
3324                                 copy = len;
3325                         elt += __skb_to_sgvec(frag_iter, sg+elt, offset - start,
3326                                               copy);
3327                         if ((len -= copy) == 0)
3328                                 return elt;
3329                         offset += copy;
3330                 }
3331                 start = end;
3332         }
3333         BUG_ON(len);
3334         return elt;
3335 }
3336 
3337 /* As compared with skb_to_sgvec, skb_to_sgvec_nomark only map skb to given
3338  * sglist without mark the sg which contain last skb data as the end.
3339  * So the caller can mannipulate sg list as will when padding new data after
3340  * the first call without calling sg_unmark_end to expend sg list.
3341  *
3342  * Scenario to use skb_to_sgvec_nomark:
3343  * 1. sg_init_table
3344  * 2. skb_to_sgvec_nomark(payload1)
3345  * 3. skb_to_sgvec_nomark(payload2)
3346  *
3347  * This is equivalent to:
3348  * 1. sg_init_table
3349  * 2. skb_to_sgvec(payload1)
3350  * 3. sg_unmark_end
3351  * 4. skb_to_sgvec(payload2)
3352  *
3353  * When mapping mutilple payload conditionally, skb_to_sgvec_nomark
3354  * is more preferable.
3355  */
3356 int skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg,
3357                         int offset, int len)
3358 {
3359         return __skb_to_sgvec(skb, sg, offset, len);
3360 }
3361 EXPORT_SYMBOL_GPL(skb_to_sgvec_nomark);
3362 
3363 int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len)
3364 {
3365         int nsg = __skb_to_sgvec(skb, sg, offset, len);
3366 
3367         sg_mark_end(&sg[nsg - 1]);
3368 
3369         return nsg;
3370 }
3371 EXPORT_SYMBOL_GPL(skb_to_sgvec);
3372 
3373 /**
3374  *      skb_cow_data - Check that a socket buffer's data buffers are writable
3375  *      @skb: The socket buffer to check.
3376  *      @tailbits: Amount of trailing space to be added
3377  *      @trailer: Returned pointer to the skb where the @tailbits space begins
3378  *
3379  *      Make sure that the data buffers attached to a socket buffer are
3380  *      writable. If they are not, private copies are made of the data buffers
3381  *      and the socket buffer is set to use these instead.
3382  *
3383  *      If @tailbits is given, make sure that there is space to write @tailbits
3384  *      bytes of data beyond current end of socket buffer.  @trailer will be
3385  *      set to point to the skb in which this space begins.
3386  *
3387  *      The number of scatterlist elements required to completely map the
3388  *      COW'd and extended socket buffer will be returned.
3389  */
3390 int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer)
3391 {
3392         int copyflag;
3393         int elt;
3394         struct sk_buff *skb1, **skb_p;
3395 
3396         /* If skb is cloned or its head is paged, reallocate
3397          * head pulling out all the pages (pages are considered not writable
3398          * at the moment even if they are anonymous).
3399          */
3400         if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) &&
3401             __pskb_pull_tail(skb, skb_pagelen(skb)-skb_headlen(skb)) == NULL)
3402                 return -ENOMEM;
3403 
3404         /* Easy case. Most of packets will go this way. */
3405         if (!skb_has_frag_list(skb)) {
3406                 /* A little of trouble, not enough of space for trailer.
3407                  * This should not happen, when stack is tuned to generate
3408                  * good frames. OK, on miss we reallocate and reserve even more
3409                  * space, 128 bytes is fair. */
3410 
3411                 if (skb_tailroom(skb) < tailbits &&
3412                     pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC))
3413                         return -ENOMEM;
3414 
3415                 /* Voila! */
3416                 *trailer = skb;
3417                 return 1;
3418         }
3419 
3420         /* Misery. We are in troubles, going to mincer fragments... */
3421 
3422         elt = 1;
3423         skb_p = &skb_shinfo(skb)->frag_list;
3424         copyflag = 0;
3425 
3426         while ((skb1 = *skb_p) != NULL) {
3427                 int ntail = 0;
3428 
3429                 /* The fragment is partially pulled by someone,
3430                  * this can happen on input. Copy it and everything
3431                  * after it. */
3432 
3433                 if (skb_shared(skb1))
3434                         copyflag = 1;
3435 
3436                 /* If the skb is the last, worry about trailer. */
3437 
3438                 if (skb1->next == NULL && tailbits) {
3439                         if (skb_shinfo(skb1)->nr_frags ||
3440                             skb_has_frag_list(skb1) ||
3441                             skb_tailroom(skb1) < tailbits)
3442                                 ntail = tailbits + 128;
3443                 }
3444 
3445                 if (copyflag ||
3446                     skb_cloned(skb1) ||
3447                     ntail ||
3448                     skb_shinfo(skb1)->nr_frags ||
3449                     skb_has_frag_list(skb1)) {
3450                         struct sk_buff *skb2;
3451 
3452                         /* Fuck, we are miserable poor guys... */
3453                         if (ntail == 0)
3454                                 skb2 = skb_copy(skb1, GFP_ATOMIC);
3455                         else
3456                                 skb2 = skb_copy_expand(skb1,
3457                                                        skb_headroom(skb1),
3458                                                        ntail,
3459                                                        GFP_ATOMIC);
3460                         if (unlikely(skb2 == NULL))
3461                                 return -ENOMEM;
3462 
3463                         if (skb1->sk)
3464                                 skb_set_owner_w(skb2, skb1->sk);
3465 
3466                         /* Looking around. Are we still alive?
3467                          * OK, link new skb, drop old one */
3468 
3469                         skb2->next = skb1->next;
3470                         *skb_p = skb2;
3471                         kfree_skb(skb1);
3472                         skb1 = skb2;
3473                 }
3474                 elt++;
3475                 *trailer = skb1;
3476                 skb_p = &skb1->next;
3477         }
3478 
3479         return elt;
3480 }
3481 EXPORT_SYMBOL_GPL(skb_cow_data);
3482 
3483 static void sock_rmem_free(struct sk_buff *skb)
3484 {
3485         struct sock *sk = skb->sk;
3486 
3487         atomic_sub(skb->truesize, &sk->sk_rmem_alloc);
3488 }
3489 
3490 /*
3491  * Note: We dont mem charge error packets (no sk_forward_alloc changes)
3492  */
3493 int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb)
3494 {
3495         if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >=
3496             (unsigned int)sk->sk_rcvbuf)
3497                 return -ENOMEM;
3498 
3499         skb_orphan(skb);
3500         skb->sk = sk;
3501         skb->destructor = sock_rmem_free;
3502         atomic_add(skb->truesize, &sk->sk_rmem_alloc);
3503 
3504         /* before exiting rcu section, make sure dst is refcounted */
3505         skb_dst_force(skb);
3506 
3507         skb_queue_tail(&sk->sk_error_queue, skb);
3508         if (!sock_flag(sk, SOCK_DEAD))
3509                 sk->sk_data_ready(sk);
3510         return 0;
3511 }
3512 EXPORT_SYMBOL(sock_queue_err_skb);
3513 
3514 struct sk_buff *sock_dequeue_err_skb(struct sock *sk)
3515 {
3516         struct sk_buff_head *q = &sk->sk_error_queue;
3517         struct sk_buff *skb, *skb_next;
3518         int err = 0;
3519 
3520         spin_lock_bh(&q->lock);
3521         skb = __skb_dequeue(q);
3522         if (skb && (skb_next = skb_peek(q)))
3523                 err = SKB_EXT_ERR(skb_next)->ee.ee_errno;
3524         spin_unlock_bh(&q->lock);
3525 
3526         sk->sk_err = err;
3527         if (err)
3528                 sk->sk_error_report(sk);
3529 
3530         return skb;
3531 }
3532 EXPORT_SYMBOL(sock_dequeue_err_skb);
3533 
3534 /**
3535  * skb_clone_sk - create clone of skb, and take reference to socket
3536  * @skb: the skb to clone
3537  *
3538  * This function creates a clone of a buffer that holds a reference on
3539  * sk_refcnt.  Buffers created via this function are meant to be
3540  * returned using sock_queue_err_skb, or free via kfree_skb.
3541  *
3542  * When passing buffers allocated with this function to sock_queue_err_skb
3543  * it is necessary to wrap the call with sock_hold/sock_put in order to
3544  * prevent the socket from being released prior to being enqueued on
3545  * the sk_error_queue.
3546  */
3547 struct sk_buff *skb_clone_sk(struct sk_buff *skb)
3548 {
3549         struct sock *sk = skb->sk;
3550         struct sk_buff *clone;
3551 
3552         if (!sk || !atomic_inc_not_zero(&sk->sk_refcnt))
3553                 return NULL;
3554 
3555         clone = skb_clone(skb, GFP_ATOMIC);
3556         if (!clone) {
3557                 sock_put(sk);
3558                 return NULL;
3559         }
3560 
3561         clone->sk = sk;
3562         clone->destructor = sock_efree;
3563 
3564         return clone;
3565 }
3566 EXPORT_SYMBOL(skb_clone_sk);
3567 
3568 static void __skb_complete_tx_timestamp(struct sk_buff *skb,
3569                                         struct sock *sk,
3570                                         int tstype)
3571 {
3572         struct sock_exterr_skb *serr;
3573         int err;
3574 
3575         serr = SKB_EXT_ERR(skb);
3576         memset(serr, 0, sizeof(*serr));
3577         serr->ee.ee_errno = ENOMSG;
3578         serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING;
3579         serr->ee.ee_info = tstype;
3580         if (sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID) {
3581                 serr->ee.ee_data = skb_shinfo(skb)->tskey;
3582                 if (sk->sk_protocol == IPPROTO_TCP)
3583                         serr->ee.ee_data -= sk->sk_tskey;
3584         }
3585 
3586         err = sock_queue_err_skb(sk, skb);
3587 
3588         if (err)
3589                 kfree_skb(skb);
3590 }
3591 
3592 void skb_complete_tx_timestamp(struct sk_buff *skb,
3593                                struct skb_shared_hwtstamps *hwtstamps)
3594 {
3595         struct sock *sk = skb->sk;
3596 
3597         /* take a reference to prevent skb_orphan() from freeing the socket */
3598         sock_hold(sk);
3599 
3600         *skb_hwtstamps(skb) = *hwtstamps;
3601         __skb_complete_tx_timestamp(skb, sk, SCM_TSTAMP_SND);
3602 
3603         sock_put(sk);
3604 }
3605 EXPORT_SYMBOL_GPL(skb_complete_tx_timestamp);
3606 
3607 void __skb_tstamp_tx(struct sk_buff *orig_skb,
3608                      struct skb_shared_hwtstamps *hwtstamps,
3609                      struct sock *sk, int tstype)
3610 {
3611         struct sk_buff *skb;
3612 
3613         if (!sk)
3614                 return;
3615 
3616         if (hwtstamps)
3617                 *skb_hwtstamps(orig_skb) = *hwtstamps;
3618         else
3619                 orig_skb->tstamp = ktime_get_real();
3620 
3621         skb = skb_clone(orig_skb, GFP_ATOMIC);
3622         if (!skb)
3623                 return;
3624 
3625         __skb_complete_tx_timestamp(skb, sk, tstype);
3626 }
3627 EXPORT_SYMBOL_GPL(__skb_tstamp_tx);
3628 
3629 void skb_tstamp_tx(struct sk_buff *orig_skb,
3630                    struct skb_shared_hwtstamps *hwtstamps)
3631 {
3632         return __skb_tstamp_tx(orig_skb, hwtstamps, orig_skb->sk,
3633                                SCM_TSTAMP_SND);
3634 }
3635 EXPORT_SYMBOL_GPL(skb_tstamp_tx);
3636 
3637 void skb_complete_wifi_ack(struct sk_buff *skb, bool acked)
3638 {
3639         struct sock *sk = skb->sk;
3640         struct sock_exterr_skb *serr;
3641         int err;
3642 
3643         skb->wifi_acked_valid = 1;
3644         skb->wifi_acked = acked;
3645 
3646         serr = SKB_EXT_ERR(skb);
3647         memset(serr, 0, sizeof(*serr));
3648         serr->ee.ee_errno = ENOMSG;
3649         serr->ee.ee_origin = SO_EE_ORIGIN_TXSTATUS;
3650 
3651         /* take a reference to prevent skb_orphan() from freeing the socket */
3652         sock_hold(sk);
3653 
3654         err = sock_queue_err_skb(sk, skb);
3655         if (err)
3656                 kfree_skb(skb);
3657 
3658         sock_put(sk);
3659 }
3660 EXPORT_SYMBOL_GPL(skb_complete_wifi_ack);
3661 
3662 
3663 /**
3664  * skb_partial_csum_set - set up and verify partial csum values for packet
3665  * @skb: the skb to set
3666  * @start: the number of bytes after skb->data to start checksumming.
3667  * @off: the offset from start to place the checksum.
3668  *
3669  * For untrusted partially-checksummed packets, we need to make sure the values
3670  * for skb->csum_start and skb->csum_offset are valid so we don't oops.
3671  *
3672  * This function checks and sets those values and skb->ip_summed: if this
3673  * returns false you should drop the packet.
3674  */
3675 bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off)
3676 {
3677         if (unlikely(start > skb_headlen(skb)) ||
3678             unlikely((int)start + off > skb_headlen(skb) - 2)) {
3679                 net_warn_ratelimited("bad partial csum: csum=%u/%u len=%u\n",
3680                                      start, off, skb_headlen(skb));
3681                 return false;
3682         }
3683         skb->ip_summed = CHECKSUM_PARTIAL;
3684         skb->csum_start = skb_headroom(skb) + start;
3685         skb->csum_offset = off;
3686         skb_set_transport_header(skb, start);
3687         return true;
3688 }
3689 EXPORT_SYMBOL_GPL(skb_partial_csum_set);
3690 
3691 static int skb_maybe_pull_tail(struct sk_buff *skb, unsigned int len,
3692                                unsigned int max)
3693 {
3694         if (skb_headlen(skb) >= len)
3695                 return 0;
3696 
3697         /* If we need to pullup then pullup to the max, so we
3698          * won't need to do it again.
3699          */
3700         if (max > skb->len)
3701                 max = skb->len;
3702 
3703         if (__pskb_pull_tail(skb, max - skb_headlen(skb)) == NULL)
3704                 return -ENOMEM;
3705 
3706         if (skb_headlen(skb) < len)
3707                 return -EPROTO;
3708 
3709         return 0;
3710 }
3711 
3712 #define MAX_TCP_HDR_LEN (15 * 4)
3713 
3714 static __sum16 *skb_checksum_setup_ip(struct sk_buff *skb,
3715                                       typeof(IPPROTO_IP) proto,
3716                                       unsigned int off)
3717 {
3718         switch (proto) {
3719                 int err;
3720 
3721         case IPPROTO_TCP:
3722                 err = skb_maybe_pull_tail(skb, off + sizeof(struct tcphdr),
3723                                           off + MAX_TCP_HDR_LEN);
3724                 if (!err && !skb_partial_csum_set(skb, off,
3725                                                   offsetof(struct tcphdr,
3726                                                            check)))
3727                         err = -EPROTO;
3728                 return err ? ERR_PTR(err) : &tcp_hdr(skb)->check;
3729 
3730         case IPPROTO_UDP:
3731                 err = skb_maybe_pull_tail(skb, off + sizeof(struct udphdr),
3732                                           off + sizeof(struct udphdr));
3733                 if (!err && !skb_partial_csum_set(skb, off,
3734                                                   offsetof(struct udphdr,
3735                                                            check)))
3736                         err = -EPROTO;
3737                 return err ? ERR_PTR(err) : &udp_hdr(skb)->check;
3738         }
3739 
3740         return ERR_PTR(-EPROTO);
3741 }
3742 
3743 /* This value should be large enough to cover a tagged ethernet header plus
3744  * maximally sized IP and TCP or UDP headers.
3745  */
3746 #define MAX_IP_HDR_LEN 128
3747 
3748 static int skb_checksum_setup_ipv4(struct sk_buff *skb, bool recalculate)
3749 {
3750         unsigned int off;
3751         bool fragment;
3752         __sum16 *csum;
3753         int err;
3754 
3755         fragment = false;
3756 
3757         err = skb_maybe_pull_tail(skb,
3758                                   sizeof(struct iphdr),
3759                                   MAX_IP_HDR_LEN);
3760         if (err < 0)
3761                 goto out;
3762 
3763         if (ip_hdr(skb)->frag_off & htons(IP_OFFSET | IP_MF))
3764                 fragment = true;
3765 
3766         off = ip_hdrlen(skb);
3767 
3768         err = -EPROTO;
3769 
3770         if (fragment)
3771                 goto out;
3772 
3773         csum = skb_checksum_setup_ip(skb, ip_hdr(skb)->protocol, off);
3774         if (IS_ERR(csum))
3775                 return PTR_ERR(csum);
3776 
3777         if (recalculate)
3778                 *csum = ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
3779                                            ip_hdr(skb)->daddr,
3780                                            skb->len - off,
3781                                            ip_hdr(skb)->protocol, 0);
3782         err = 0;
3783 
3784 out:
3785         return err;
3786 }
3787 
3788 /* This value should be large enough to cover a tagged ethernet header plus
3789  * an IPv6 header, all options, and a maximal TCP or UDP header.
3790  */
3791 #define MAX_IPV6_HDR_LEN 256
3792 
3793 #define OPT_HDR(type, skb, off) \
3794         (type *)(skb_network_header(skb) + (off))
3795 
3796 static int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate)
3797 {
3798         int err;
3799         u8 nexthdr;
3800         unsigned int off;
3801         unsigned int len;
3802         bool fragment;
3803         bool done;
3804         __sum16 *csum;
3805 
3806         fragment = false;
3807         done = false;
3808 
3809         off = sizeof(struct ipv6hdr);
3810 
3811         err = skb_maybe_pull_tail(skb, off, MAX_IPV6_HDR_LEN);
3812         if (err < 0)
3813                 goto out;
3814 
3815         nexthdr = ipv6_hdr(skb)->nexthdr;
3816 
3817         len = sizeof(struct ipv6hdr) + ntohs(ipv6_hdr(skb)->payload_len);
3818         while (off <= len && !done) {
3819                 switch (nexthdr) {
3820                 case IPPROTO_DSTOPTS:
3821                 case IPPROTO_HOPOPTS:
3822                 case IPPROTO_ROUTING: {
3823                         struct ipv6_opt_hdr *hp;
3824 
3825                         err = skb_maybe_pull_tail(skb,
3826                                                   off +
3827                                                   sizeof(struct ipv6_opt_hdr),
3828                                                   MAX_IPV6_HDR_LEN);
3829                         if (err < 0)
3830                                 goto out;
3831 
3832                         hp = OPT_HDR(struct ipv6_opt_hdr, skb, off);
3833                         nexthdr = hp->nexthdr;
3834                         off += ipv6_optlen(hp);
3835                         break;
3836                 }
3837                 case IPPROTO_AH: {
3838                         struct ip_auth_hdr *hp;
3839 
3840                         err = skb_maybe_pull_tail(skb,
3841                                                   off +
3842                                                   sizeof(struct ip_auth_hdr),
3843                                                   MAX_IPV6_HDR_LEN);
3844                         if (err < 0)
3845                                 goto out;
3846 
3847                         hp = OPT_HDR(struct ip_auth_hdr, skb, off);
3848                         nexthdr = hp->nexthdr;
3849                         off += ipv6_authlen(hp);
3850                         break;
3851                 }
3852                 case IPPROTO_FRAGMENT: {
3853                         struct frag_hdr *hp;
3854 
3855                         err = skb_maybe_pull_tail(skb,
3856                                                   off +
3857                                                   sizeof(struct frag_hdr),
3858                                                   MAX_IPV6_HDR_LEN);
3859                         if (err < 0)
3860                                 goto out;
3861 
3862                         hp = OPT_HDR(struct frag_hdr, skb, off);
3863 
3864                         if (hp->frag_off & htons(IP6_OFFSET | IP6_MF))
3865                                 fragment = true;
3866 
3867                         nexthdr = hp->nexthdr;
3868                         off += sizeof(struct frag_hdr);
3869                         break;
3870                 }
3871                 default:
3872                         done = true;
3873                         break;
3874                 }
3875         }
3876 
3877         err = -EPROTO;
3878 
3879         if (!done || fragment)
3880                 goto out;
3881 
3882         csum = skb_checksum_setup_ip(skb, nexthdr, off);
3883         if (IS_ERR(csum))
3884                 return PTR_ERR(csum);
3885 
3886         if (recalculate)
3887                 *csum = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3888                                          &ipv6_hdr(skb)->daddr,
3889                                          skb->len - off, nexthdr, 0);
3890         err = 0;
3891 
3892 out:
3893         return err;
3894 }
3895 
3896 /**
3897  * skb_checksum_setup - set up partial checksum offset
3898  * @skb: the skb to set up
3899  * @recalculate: if true the pseudo-header checksum will be recalculated
3900  */
3901 int skb_checksum_setup(struct sk_buff *skb, bool recalculate)
3902 {
3903         int err;
3904 
3905         switch (skb->protocol) {
3906         case htons(ETH_P_IP):
3907                 err = skb_checksum_setup_ipv4(skb, recalculate);
3908                 break;
3909 
3910         case htons(ETH_P_IPV6):
3911                 err = skb_checksum_setup_ipv6(skb, recalculate);
3912                 break;
3913 
3914         default:
3915                 err = -EPROTO;
3916                 break;
3917         }
3918 
3919         return err;
3920 }
3921 EXPORT_SYMBOL(skb_checksum_setup);
3922 
3923 void __skb_warn_lro_forwarding(const struct sk_buff *skb)
3924 {
3925         net_warn_ratelimited("%s: received packets cannot be forwarded while LRO is enabled\n",
3926                              skb->dev->name);
3927 }
3928 EXPORT_SYMBOL(__skb_warn_lro_forwarding);
3929 
3930 void kfree_skb_partial(struct sk_buff *skb, bool head_stolen)
3931 {
3932         if (head_stolen) {
3933                 skb_release_head_state(skb);
3934                 kmem_cache_free(skbuff_head_cache, skb);
3935         } else {
3936                 __kfree_skb(skb);
3937         }
3938 }
3939 EXPORT_SYMBOL(kfree_skb_partial);
3940 
3941 /**
3942  * skb_try_coalesce - try to merge skb to prior one
3943  * @to: prior buffer
3944  * @from: buffer to add
3945  * @fragstolen: pointer to boolean
3946  * @delta_truesize: how much more was allocated than was requested
3947  */
3948 bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
3949                       bool *fragstolen, int *delta_truesize)
3950 {
3951         int i, delta, len = from->len;
3952 
3953         *fragstolen = false;
3954 
3955         if (skb_cloned(to))
3956                 return false;
3957 
3958         if (len <= skb_tailroom(to)) {
3959                 if (len)
3960                         BUG_ON(skb_copy_bits(from, 0, skb_put(to, len), len));
3961                 *delta_truesize = 0;
3962                 return true;
3963         }
3964 
3965         if (skb_has_frag_list(to) || skb_has_frag_list(from))
3966                 return false;
3967 
3968         if (skb_headlen(from) != 0) {
3969                 struct page *page;
3970                 unsigned int offset;
3971 
3972                 if (skb_shinfo(to)->nr_frags +
3973                     skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS)
3974                         return false;
3975 
3976                 if (skb_head_is_locked(from))
3977                         return false;
3978 
3979                 delta = from->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff));
3980 
3981                 page = virt_to_head_page(from->head);
3982                 offset = from->data - (unsigned char *)page_address(page);
3983 
3984                 skb_fill_page_desc(to, skb_shinfo(to)->nr_frags,
3985                                    page, offset, skb_headlen(from));
3986                 *fragstolen = true;
3987         } else {
3988                 if (skb_shinfo(to)->nr_frags +
3989                     skb_shinfo(from)->nr_frags > MAX_SKB_FRAGS)
3990                         return false;
3991 
3992                 delta = from->truesize - SKB_TRUESIZE(skb_end_offset(from));
3993         }
3994 
3995         WARN_ON_ONCE(delta < len);
3996 
3997         memcpy(skb_shinfo(to)->frags + skb_shinfo(to)->nr_frags,
3998                skb_shinfo(from)->frags,
3999                skb_shinfo(from)->nr_frags * sizeof(skb_frag_t));
4000         skb_shinfo(to)->nr_frags += skb_shinfo(from)->nr_frags;
4001 
4002         if (!skb_cloned(from))
4003                 skb_shinfo(from)->nr_frags = 0;
4004 
4005         /* if the skb is not cloned this does nothing
4006          * since we set nr_frags to 0.
4007          */
4008         for (i = 0; i < skb_shinfo(from)->nr_frags; i++)
4009                 skb_frag_ref(from, i);
4010 
4011         to->truesize += delta;
4012         to->len += len;
4013         to->data_len += len;
4014 
4015         *delta_truesize = delta;
4016         return true;
4017 }
4018 EXPORT_SYMBOL(skb_try_coalesce);
4019 
4020 /**
4021  * skb_scrub_packet - scrub an skb
4022  *
4023  * @skb: buffer to clean
4024  * @xnet: packet is crossing netns
4025  *
4026  * skb_scrub_packet can be used after encapsulating or decapsulting a packet
4027  * into/from a tunnel. Some information have to be cleared during these
4028  * operations.
4029  * skb_scrub_packet can also be used to clean a skb before injecting it in
4030  * another namespace (@xnet == true). We have to clear all information in the
4031  * skb that could impact namespace isolation.
4032  */
4033 void skb_scrub_packet(struct sk_buff *skb, bool xnet)
4034 {
4035         if (xnet)
4036                 skb_orphan(skb);
4037         skb->tstamp.tv64 = 0;
4038         skb->pkt_type = PACKET_HOST;
4039         skb->skb_iif = 0;
4040         skb->ignore_df = 0;
4041         skb_dst_drop(skb);
4042         skb->mark = 0;
4043         secpath_reset(skb);
4044         nf_reset(skb);
4045         nf_reset_trace(skb);
4046 }
4047 EXPORT_SYMBOL_GPL(skb_scrub_packet);
4048 
4049 /**
4050  * skb_gso_transport_seglen - Return length of individual segments of a gso packet
4051  *
4052  * @skb: GSO skb
4053  *
4054  * skb_gso_transport_seglen is used to determine the real size of the
4055  * individual segments, including Layer4 headers (TCP/UDP).
4056  *
4057  * The MAC/L2 or network (IP, IPv6) headers are not accounted for.
4058  */
4059 unsigned int skb_gso_transport_seglen(const struct sk_buff *skb)
4060 {
4061         const struct skb_shared_info *shinfo = skb_shinfo(skb);
4062         unsigned int thlen = 0;
4063 
4064         if (skb->encapsulation) {
4065                 thlen = skb_inner_transport_header(skb) -
4066                         skb_transport_header(skb);
4067 
4068                 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
4069                         thlen += inner_tcp_hdrlen(skb);
4070         } else if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) {
4071                 thlen = tcp_hdrlen(skb);
4072         }
4073         /* UFO sets gso_size to the size of the fragmentation
4074          * payload, i.e. the size of the L4 (UDP) header is already
4075          * accounted for.
4076          */
4077         return thlen + shinfo->gso_size;
4078 }
4079 EXPORT_SYMBOL_GPL(skb_gso_transport_seglen);
4080 
4081 static struct sk_buff *skb_reorder_vlan_header(struct sk_buff *skb)
4082 {
4083         if (skb_cow(skb, skb_headroom(skb)) < 0) {
4084                 kfree_skb(skb);
4085                 return NULL;
4086         }
4087 
4088         memmove(skb->data - ETH_HLEN, skb->data - VLAN_ETH_HLEN, 2 * ETH_ALEN);
4089         skb->mac_header += VLAN_HLEN;
4090         return skb;
4091 }
4092 
4093 struct sk_buff *skb_vlan_untag(struct sk_buff *skb)
4094 {
4095         struct vlan_hdr *vhdr;
4096         u16 vlan_tci;
4097 
4098         if (unlikely(vlan_tx_tag_present(skb))) {
4099                 /* vlan_tci is already set-up so leave this for another time */
4100                 return skb;
4101         }
4102 
4103         skb = skb_share_check(skb, GFP_ATOMIC);
4104         if (unlikely(!skb))
4105                 goto err_free;
4106 
4107         if (unlikely(!pskb_may_pull(skb, VLAN_HLEN)))
4108                 goto err_free;
4109 
4110         vhdr = (struct vlan_hdr *)skb->data;
4111         vlan_tci = ntohs(vhdr->h_vlan_TCI);
4112         __vlan_hwaccel_put_tag(skb, skb->protocol, vlan_tci);
4113 
4114         skb_pull_rcsum(skb, VLAN_HLEN);
4115         vlan_set_encap_proto(skb, vhdr);
4116 
4117         skb = skb_reorder_vlan_header(skb);
4118         if (unlikely(!skb))
4119                 goto err_free;
4120 
4121         skb_reset_network_header(skb);
4122         skb_reset_transport_header(skb);
4123         skb_reset_mac_len(skb);
4124 
4125         return skb;
4126 
4127 err_free:
4128         kfree_skb(skb);
4129         return NULL;
4130 }
4131 EXPORT_SYMBOL(skb_vlan_untag);
4132 
4133 /**
4134  * alloc_skb_with_frags - allocate skb with page frags
4135  *
4136  * @header_len: size of linear part
4137  * @data_len: needed length in frags
4138  * @max_page_order: max page order desired.
4139  * @errcode: pointer to error code if any
4140  * @gfp_mask: allocation mask
4141  *
4142  * This can be used to allocate a paged skb, given a maximal order for frags.
4143  */
4144 struct sk_buff *alloc_skb_with_frags(unsigned long header_len,
4145                                      unsigned long data_len,
4146                                      int max_page_order,
4147                                      int *errcode,
4148                                      gfp_t gfp_mask)
4149 {
4150         int npages = (data_len + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
4151         unsigned long chunk;
4152         struct sk_buff *skb;
4153         struct page *page;
4154         gfp_t gfp_head;
4155         int i;
4156 
4157         *errcode = -EMSGSIZE;
4158         /* Note this test could be relaxed, if we succeed to allocate
4159          * high order pages...
4160          */
4161         if (npages > MAX_SKB_FRAGS)
4162                 return NULL;
4163 
4164         gfp_head = gfp_mask;
4165         if (gfp_head & __GFP_WAIT)
4166                 gfp_head |= __GFP_REPEAT;
4167 
4168         *errcode = -ENOBUFS;
4169         skb = alloc_skb(header_len, gfp_head);
4170         if (!skb)
4171                 return NULL;
4172 
4173         skb->truesize += npages << PAGE_SHIFT;
4174 
4175         for (i = 0; npages > 0; i++) {
4176                 int order = max_page_order;
4177 
4178                 while (order) {
4179                         if (npages >= 1 << order) {
4180                                 page = alloc_pages(gfp_mask |
4181                                                    __GFP_COMP |
4182                                                    __GFP_NOWARN |
4183                                                    __GFP_NORETRY,
4184                                                    order);
4185                                 if (page)
4186                                         goto fill_page;
4187                                 /* Do not retry other high order allocations */
4188                                 order = 1;
4189                                 max_page_order = 0;
4190                         }
4191                         order--;
4192                 }
4193                 page = alloc_page(gfp_mask);
4194                 if (!page)
4195                         goto failure;
4196 fill_page:
4197                 chunk = min_t(unsigned long, data_len,
4198                               PAGE_SIZE << order);
4199                 skb_fill_page_desc(skb, i, page, 0, chunk);
4200                 data_len -= chunk;
4201                 npages -= 1 << order;
4202         }
4203         return skb;
4204 
4205 failure:
4206         kfree_skb(skb);
4207         return NULL;
4208 }
4209 EXPORT_SYMBOL(alloc_skb_with_frags);
4210 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us