Version:  2.0.40 2.2.26 2.4.37 3.0 3.1 3.2 3.3 3.4 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16

Linux/net/core/skbuff.c

  1 /*
  2  *      Routines having to do with the 'struct sk_buff' memory handlers.
  3  *
  4  *      Authors:        Alan Cox <alan@lxorguk.ukuu.org.uk>
  5  *                      Florian La Roche <rzsfl@rz.uni-sb.de>
  6  *
  7  *      Fixes:
  8  *              Alan Cox        :       Fixed the worst of the load
  9  *                                      balancer bugs.
 10  *              Dave Platt      :       Interrupt stacking fix.
 11  *      Richard Kooijman        :       Timestamp fixes.
 12  *              Alan Cox        :       Changed buffer format.
 13  *              Alan Cox        :       destructor hook for AF_UNIX etc.
 14  *              Linus Torvalds  :       Better skb_clone.
 15  *              Alan Cox        :       Added skb_copy.
 16  *              Alan Cox        :       Added all the changed routines Linus
 17  *                                      only put in the headers
 18  *              Ray VanTassle   :       Fixed --skb->lock in free
 19  *              Alan Cox        :       skb_copy copy arp field
 20  *              Andi Kleen      :       slabified it.
 21  *              Robert Olsson   :       Removed skb_head_pool
 22  *
 23  *      NOTE:
 24  *              The __skb_ routines should be called with interrupts
 25  *      disabled, or you better be *real* sure that the operation is atomic
 26  *      with respect to whatever list is being frobbed (e.g. via lock_sock()
 27  *      or via disabling bottom half handlers, etc).
 28  *
 29  *      This program is free software; you can redistribute it and/or
 30  *      modify it under the terms of the GNU General Public License
 31  *      as published by the Free Software Foundation; either version
 32  *      2 of the License, or (at your option) any later version.
 33  */
 34 
 35 /*
 36  *      The functions in this file will not compile correctly with gcc 2.4.x
 37  */
 38 
 39 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 40 
 41 #include <linux/module.h>
 42 #include <linux/types.h>
 43 #include <linux/kernel.h>
 44 #include <linux/kmemcheck.h>
 45 #include <linux/mm.h>
 46 #include <linux/interrupt.h>
 47 #include <linux/in.h>
 48 #include <linux/inet.h>
 49 #include <linux/slab.h>
 50 #include <linux/tcp.h>
 51 #include <linux/udp.h>
 52 #include <linux/netdevice.h>
 53 #ifdef CONFIG_NET_CLS_ACT
 54 #include <net/pkt_sched.h>
 55 #endif
 56 #include <linux/string.h>
 57 #include <linux/skbuff.h>
 58 #include <linux/splice.h>
 59 #include <linux/cache.h>
 60 #include <linux/rtnetlink.h>
 61 #include <linux/init.h>
 62 #include <linux/scatterlist.h>
 63 #include <linux/errqueue.h>
 64 #include <linux/prefetch.h>
 65 
 66 #include <net/protocol.h>
 67 #include <net/dst.h>
 68 #include <net/sock.h>
 69 #include <net/checksum.h>
 70 #include <net/ip6_checksum.h>
 71 #include <net/xfrm.h>
 72 
 73 #include <asm/uaccess.h>
 74 #include <trace/events/skb.h>
 75 #include <linux/highmem.h>
 76 
 77 struct kmem_cache *skbuff_head_cache __read_mostly;
 78 static struct kmem_cache *skbuff_fclone_cache __read_mostly;
 79 
 80 /**
 81  *      skb_panic - private function for out-of-line support
 82  *      @skb:   buffer
 83  *      @sz:    size
 84  *      @addr:  address
 85  *      @msg:   skb_over_panic or skb_under_panic
 86  *
 87  *      Out-of-line support for skb_put() and skb_push().
 88  *      Called via the wrapper skb_over_panic() or skb_under_panic().
 89  *      Keep out of line to prevent kernel bloat.
 90  *      __builtin_return_address is not used because it is not always reliable.
 91  */
 92 static void skb_panic(struct sk_buff *skb, unsigned int sz, void *addr,
 93                       const char msg[])
 94 {
 95         pr_emerg("%s: text:%p len:%d put:%d head:%p data:%p tail:%#lx end:%#lx dev:%s\n",
 96                  msg, addr, skb->len, sz, skb->head, skb->data,
 97                  (unsigned long)skb->tail, (unsigned long)skb->end,
 98                  skb->dev ? skb->dev->name : "<NULL>");
 99         BUG();
100 }
101 
102 static void skb_over_panic(struct sk_buff *skb, unsigned int sz, void *addr)
103 {
104         skb_panic(skb, sz, addr, __func__);
105 }
106 
107 static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr)
108 {
109         skb_panic(skb, sz, addr, __func__);
110 }
111 
112 /*
113  * kmalloc_reserve is a wrapper around kmalloc_node_track_caller that tells
114  * the caller if emergency pfmemalloc reserves are being used. If it is and
115  * the socket is later found to be SOCK_MEMALLOC then PFMEMALLOC reserves
116  * may be used. Otherwise, the packet data may be discarded until enough
117  * memory is free
118  */
119 #define kmalloc_reserve(size, gfp, node, pfmemalloc) \
120          __kmalloc_reserve(size, gfp, node, _RET_IP_, pfmemalloc)
121 
122 static void *__kmalloc_reserve(size_t size, gfp_t flags, int node,
123                                unsigned long ip, bool *pfmemalloc)
124 {
125         void *obj;
126         bool ret_pfmemalloc = false;
127 
128         /*
129          * Try a regular allocation, when that fails and we're not entitled
130          * to the reserves, fail.
131          */
132         obj = kmalloc_node_track_caller(size,
133                                         flags | __GFP_NOMEMALLOC | __GFP_NOWARN,
134                                         node);
135         if (obj || !(gfp_pfmemalloc_allowed(flags)))
136                 goto out;
137 
138         /* Try again but now we are using pfmemalloc reserves */
139         ret_pfmemalloc = true;
140         obj = kmalloc_node_track_caller(size, flags, node);
141 
142 out:
143         if (pfmemalloc)
144                 *pfmemalloc = ret_pfmemalloc;
145 
146         return obj;
147 }
148 
149 /*      Allocate a new skbuff. We do this ourselves so we can fill in a few
150  *      'private' fields and also do memory statistics to find all the
151  *      [BEEP] leaks.
152  *
153  */
154 
155 struct sk_buff *__alloc_skb_head(gfp_t gfp_mask, int node)
156 {
157         struct sk_buff *skb;
158 
159         /* Get the HEAD */
160         skb = kmem_cache_alloc_node(skbuff_head_cache,
161                                     gfp_mask & ~__GFP_DMA, node);
162         if (!skb)
163                 goto out;
164 
165         /*
166          * Only clear those fields we need to clear, not those that we will
167          * actually initialise below. Hence, don't put any more fields after
168          * the tail pointer in struct sk_buff!
169          */
170         memset(skb, 0, offsetof(struct sk_buff, tail));
171         skb->head = NULL;
172         skb->truesize = sizeof(struct sk_buff);
173         atomic_set(&skb->users, 1);
174 
175         skb->mac_header = (typeof(skb->mac_header))~0U;
176 out:
177         return skb;
178 }
179 
180 /**
181  *      __alloc_skb     -       allocate a network buffer
182  *      @size: size to allocate
183  *      @gfp_mask: allocation mask
184  *      @flags: If SKB_ALLOC_FCLONE is set, allocate from fclone cache
185  *              instead of head cache and allocate a cloned (child) skb.
186  *              If SKB_ALLOC_RX is set, __GFP_MEMALLOC will be used for
187  *              allocations in case the data is required for writeback
188  *      @node: numa node to allocate memory on
189  *
190  *      Allocate a new &sk_buff. The returned buffer has no headroom and a
191  *      tail room of at least size bytes. The object has a reference count
192  *      of one. The return is the buffer. On a failure the return is %NULL.
193  *
194  *      Buffers may only be allocated from interrupts using a @gfp_mask of
195  *      %GFP_ATOMIC.
196  */
197 struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
198                             int flags, int node)
199 {
200         struct kmem_cache *cache;
201         struct skb_shared_info *shinfo;
202         struct sk_buff *skb;
203         u8 *data;
204         bool pfmemalloc;
205 
206         cache = (flags & SKB_ALLOC_FCLONE)
207                 ? skbuff_fclone_cache : skbuff_head_cache;
208 
209         if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX))
210                 gfp_mask |= __GFP_MEMALLOC;
211 
212         /* Get the HEAD */
213         skb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node);
214         if (!skb)
215                 goto out;
216         prefetchw(skb);
217 
218         /* We do our best to align skb_shared_info on a separate cache
219          * line. It usually works because kmalloc(X > SMP_CACHE_BYTES) gives
220          * aligned memory blocks, unless SLUB/SLAB debug is enabled.
221          * Both skb->head and skb_shared_info are cache line aligned.
222          */
223         size = SKB_DATA_ALIGN(size);
224         size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
225         data = kmalloc_reserve(size, gfp_mask, node, &pfmemalloc);
226         if (!data)
227                 goto nodata;
228         /* kmalloc(size) might give us more room than requested.
229          * Put skb_shared_info exactly at the end of allocated zone,
230          * to allow max possible filling before reallocation.
231          */
232         size = SKB_WITH_OVERHEAD(ksize(data));
233         prefetchw(data + size);
234 
235         /*
236          * Only clear those fields we need to clear, not those that we will
237          * actually initialise below. Hence, don't put any more fields after
238          * the tail pointer in struct sk_buff!
239          */
240         memset(skb, 0, offsetof(struct sk_buff, tail));
241         /* Account for allocated memory : skb + skb->head */
242         skb->truesize = SKB_TRUESIZE(size);
243         skb->pfmemalloc = pfmemalloc;
244         atomic_set(&skb->users, 1);
245         skb->head = data;
246         skb->data = data;
247         skb_reset_tail_pointer(skb);
248         skb->end = skb->tail + size;
249         skb->mac_header = (typeof(skb->mac_header))~0U;
250         skb->transport_header = (typeof(skb->transport_header))~0U;
251 
252         /* make sure we initialize shinfo sequentially */
253         shinfo = skb_shinfo(skb);
254         memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
255         atomic_set(&shinfo->dataref, 1);
256         kmemcheck_annotate_variable(shinfo->destructor_arg);
257 
258         if (flags & SKB_ALLOC_FCLONE) {
259                 struct sk_buff *child = skb + 1;
260                 atomic_t *fclone_ref = (atomic_t *) (child + 1);
261 
262                 kmemcheck_annotate_bitfield(child, flags1);
263                 kmemcheck_annotate_bitfield(child, flags2);
264                 skb->fclone = SKB_FCLONE_ORIG;
265                 atomic_set(fclone_ref, 1);
266 
267                 child->fclone = SKB_FCLONE_UNAVAILABLE;
268                 child->pfmemalloc = pfmemalloc;
269         }
270 out:
271         return skb;
272 nodata:
273         kmem_cache_free(cache, skb);
274         skb = NULL;
275         goto out;
276 }
277 EXPORT_SYMBOL(__alloc_skb);
278 
279 /**
280  * build_skb - build a network buffer
281  * @data: data buffer provided by caller
282  * @frag_size: size of fragment, or 0 if head was kmalloced
283  *
284  * Allocate a new &sk_buff. Caller provides space holding head and
285  * skb_shared_info. @data must have been allocated by kmalloc() only if
286  * @frag_size is 0, otherwise data should come from the page allocator.
287  * The return is the new skb buffer.
288  * On a failure the return is %NULL, and @data is not freed.
289  * Notes :
290  *  Before IO, driver allocates only data buffer where NIC put incoming frame
291  *  Driver should add room at head (NET_SKB_PAD) and
292  *  MUST add room at tail (SKB_DATA_ALIGN(skb_shared_info))
293  *  After IO, driver calls build_skb(), to allocate sk_buff and populate it
294  *  before giving packet to stack.
295  *  RX rings only contains data buffers, not full skbs.
296  */
297 struct sk_buff *build_skb(void *data, unsigned int frag_size)
298 {
299         struct skb_shared_info *shinfo;
300         struct sk_buff *skb;
301         unsigned int size = frag_size ? : ksize(data);
302 
303         skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC);
304         if (!skb)
305                 return NULL;
306 
307         size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
308 
309         memset(skb, 0, offsetof(struct sk_buff, tail));
310         skb->truesize = SKB_TRUESIZE(size);
311         skb->head_frag = frag_size != 0;
312         atomic_set(&skb->users, 1);
313         skb->head = data;
314         skb->data = data;
315         skb_reset_tail_pointer(skb);
316         skb->end = skb->tail + size;
317         skb->mac_header = (typeof(skb->mac_header))~0U;
318         skb->transport_header = (typeof(skb->transport_header))~0U;
319 
320         /* make sure we initialize shinfo sequentially */
321         shinfo = skb_shinfo(skb);
322         memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
323         atomic_set(&shinfo->dataref, 1);
324         kmemcheck_annotate_variable(shinfo->destructor_arg);
325 
326         return skb;
327 }
328 EXPORT_SYMBOL(build_skb);
329 
330 struct netdev_alloc_cache {
331         struct page_frag        frag;
332         /* we maintain a pagecount bias, so that we dont dirty cache line
333          * containing page->_count every time we allocate a fragment.
334          */
335         unsigned int            pagecnt_bias;
336 };
337 static DEFINE_PER_CPU(struct netdev_alloc_cache, netdev_alloc_cache);
338 
339 static void *__netdev_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
340 {
341         struct netdev_alloc_cache *nc;
342         void *data = NULL;
343         int order;
344         unsigned long flags;
345 
346         local_irq_save(flags);
347         nc = &__get_cpu_var(netdev_alloc_cache);
348         if (unlikely(!nc->frag.page)) {
349 refill:
350                 for (order = NETDEV_FRAG_PAGE_MAX_ORDER; ;) {
351                         gfp_t gfp = gfp_mask;
352 
353                         if (order)
354                                 gfp |= __GFP_COMP | __GFP_NOWARN;
355                         nc->frag.page = alloc_pages(gfp, order);
356                         if (likely(nc->frag.page))
357                                 break;
358                         if (--order < 0)
359                                 goto end;
360                 }
361                 nc->frag.size = PAGE_SIZE << order;
362 recycle:
363                 atomic_set(&nc->frag.page->_count, NETDEV_PAGECNT_MAX_BIAS);
364                 nc->pagecnt_bias = NETDEV_PAGECNT_MAX_BIAS;
365                 nc->frag.offset = 0;
366         }
367 
368         if (nc->frag.offset + fragsz > nc->frag.size) {
369                 /* avoid unnecessary locked operations if possible */
370                 if ((atomic_read(&nc->frag.page->_count) == nc->pagecnt_bias) ||
371                     atomic_sub_and_test(nc->pagecnt_bias, &nc->frag.page->_count))
372                         goto recycle;
373                 goto refill;
374         }
375 
376         data = page_address(nc->frag.page) + nc->frag.offset;
377         nc->frag.offset += fragsz;
378         nc->pagecnt_bias--;
379 end:
380         local_irq_restore(flags);
381         return data;
382 }
383 
384 /**
385  * netdev_alloc_frag - allocate a page fragment
386  * @fragsz: fragment size
387  *
388  * Allocates a frag from a page for receive buffer.
389  * Uses GFP_ATOMIC allocations.
390  */
391 void *netdev_alloc_frag(unsigned int fragsz)
392 {
393         return __netdev_alloc_frag(fragsz, GFP_ATOMIC | __GFP_COLD);
394 }
395 EXPORT_SYMBOL(netdev_alloc_frag);
396 
397 /**
398  *      __netdev_alloc_skb - allocate an skbuff for rx on a specific device
399  *      @dev: network device to receive on
400  *      @length: length to allocate
401  *      @gfp_mask: get_free_pages mask, passed to alloc_skb
402  *
403  *      Allocate a new &sk_buff and assign it a usage count of one. The
404  *      buffer has unspecified headroom built in. Users should allocate
405  *      the headroom they think they need without accounting for the
406  *      built in space. The built in space is used for optimisations.
407  *
408  *      %NULL is returned if there is no free memory.
409  */
410 struct sk_buff *__netdev_alloc_skb(struct net_device *dev,
411                                    unsigned int length, gfp_t gfp_mask)
412 {
413         struct sk_buff *skb = NULL;
414         unsigned int fragsz = SKB_DATA_ALIGN(length + NET_SKB_PAD) +
415                               SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
416 
417         if (fragsz <= PAGE_SIZE && !(gfp_mask & (__GFP_WAIT | GFP_DMA))) {
418                 void *data;
419 
420                 if (sk_memalloc_socks())
421                         gfp_mask |= __GFP_MEMALLOC;
422 
423                 data = __netdev_alloc_frag(fragsz, gfp_mask);
424 
425                 if (likely(data)) {
426                         skb = build_skb(data, fragsz);
427                         if (unlikely(!skb))
428                                 put_page(virt_to_head_page(data));
429                 }
430         } else {
431                 skb = __alloc_skb(length + NET_SKB_PAD, gfp_mask,
432                                   SKB_ALLOC_RX, NUMA_NO_NODE);
433         }
434         if (likely(skb)) {
435                 skb_reserve(skb, NET_SKB_PAD);
436                 skb->dev = dev;
437         }
438         return skb;
439 }
440 EXPORT_SYMBOL(__netdev_alloc_skb);
441 
442 void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off,
443                      int size, unsigned int truesize)
444 {
445         skb_fill_page_desc(skb, i, page, off, size);
446         skb->len += size;
447         skb->data_len += size;
448         skb->truesize += truesize;
449 }
450 EXPORT_SYMBOL(skb_add_rx_frag);
451 
452 void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size,
453                           unsigned int truesize)
454 {
455         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
456 
457         skb_frag_size_add(frag, size);
458         skb->len += size;
459         skb->data_len += size;
460         skb->truesize += truesize;
461 }
462 EXPORT_SYMBOL(skb_coalesce_rx_frag);
463 
464 static void skb_drop_list(struct sk_buff **listp)
465 {
466         kfree_skb_list(*listp);
467         *listp = NULL;
468 }
469 
470 static inline void skb_drop_fraglist(struct sk_buff *skb)
471 {
472         skb_drop_list(&skb_shinfo(skb)->frag_list);
473 }
474 
475 static void skb_clone_fraglist(struct sk_buff *skb)
476 {
477         struct sk_buff *list;
478 
479         skb_walk_frags(skb, list)
480                 skb_get(list);
481 }
482 
483 static void skb_free_head(struct sk_buff *skb)
484 {
485         if (skb->head_frag)
486                 put_page(virt_to_head_page(skb->head));
487         else
488                 kfree(skb->head);
489 }
490 
491 static void skb_release_data(struct sk_buff *skb)
492 {
493         if (!skb->cloned ||
494             !atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1,
495                                &skb_shinfo(skb)->dataref)) {
496                 if (skb_shinfo(skb)->nr_frags) {
497                         int i;
498                         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
499                                 skb_frag_unref(skb, i);
500                 }
501 
502                 /*
503                  * If skb buf is from userspace, we need to notify the caller
504                  * the lower device DMA has done;
505                  */
506                 if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) {
507                         struct ubuf_info *uarg;
508 
509                         uarg = skb_shinfo(skb)->destructor_arg;
510                         if (uarg->callback)
511                                 uarg->callback(uarg, true);
512                 }
513 
514                 if (skb_has_frag_list(skb))
515                         skb_drop_fraglist(skb);
516 
517                 skb_free_head(skb);
518         }
519 }
520 
521 /*
522  *      Free an skbuff by memory without cleaning the state.
523  */
524 static void kfree_skbmem(struct sk_buff *skb)
525 {
526         struct sk_buff *other;
527         atomic_t *fclone_ref;
528 
529         switch (skb->fclone) {
530         case SKB_FCLONE_UNAVAILABLE:
531                 kmem_cache_free(skbuff_head_cache, skb);
532                 break;
533 
534         case SKB_FCLONE_ORIG:
535                 fclone_ref = (atomic_t *) (skb + 2);
536                 if (atomic_dec_and_test(fclone_ref))
537                         kmem_cache_free(skbuff_fclone_cache, skb);
538                 break;
539 
540         case SKB_FCLONE_CLONE:
541                 fclone_ref = (atomic_t *) (skb + 1);
542                 other = skb - 1;
543 
544                 /* The clone portion is available for
545                  * fast-cloning again.
546                  */
547                 skb->fclone = SKB_FCLONE_UNAVAILABLE;
548 
549                 if (atomic_dec_and_test(fclone_ref))
550                         kmem_cache_free(skbuff_fclone_cache, other);
551                 break;
552         }
553 }
554 
555 static void skb_release_head_state(struct sk_buff *skb)
556 {
557         skb_dst_drop(skb);
558 #ifdef CONFIG_XFRM
559         secpath_put(skb->sp);
560 #endif
561         if (skb->destructor) {
562                 WARN_ON(in_irq());
563                 skb->destructor(skb);
564         }
565 #if IS_ENABLED(CONFIG_NF_CONNTRACK)
566         nf_conntrack_put(skb->nfct);
567 #endif
568 #ifdef CONFIG_BRIDGE_NETFILTER
569         nf_bridge_put(skb->nf_bridge);
570 #endif
571 /* XXX: IS this still necessary? - JHS */
572 #ifdef CONFIG_NET_SCHED
573         skb->tc_index = 0;
574 #ifdef CONFIG_NET_CLS_ACT
575         skb->tc_verd = 0;
576 #endif
577 #endif
578 }
579 
580 /* Free everything but the sk_buff shell. */
581 static void skb_release_all(struct sk_buff *skb)
582 {
583         skb_release_head_state(skb);
584         if (likely(skb->head))
585                 skb_release_data(skb);
586 }
587 
588 /**
589  *      __kfree_skb - private function
590  *      @skb: buffer
591  *
592  *      Free an sk_buff. Release anything attached to the buffer.
593  *      Clean the state. This is an internal helper function. Users should
594  *      always call kfree_skb
595  */
596 
597 void __kfree_skb(struct sk_buff *skb)
598 {
599         skb_release_all(skb);
600         kfree_skbmem(skb);
601 }
602 EXPORT_SYMBOL(__kfree_skb);
603 
604 /**
605  *      kfree_skb - free an sk_buff
606  *      @skb: buffer to free
607  *
608  *      Drop a reference to the buffer and free it if the usage count has
609  *      hit zero.
610  */
611 void kfree_skb(struct sk_buff *skb)
612 {
613         if (unlikely(!skb))
614                 return;
615         if (likely(atomic_read(&skb->users) == 1))
616                 smp_rmb();
617         else if (likely(!atomic_dec_and_test(&skb->users)))
618                 return;
619         trace_kfree_skb(skb, __builtin_return_address(0));
620         __kfree_skb(skb);
621 }
622 EXPORT_SYMBOL(kfree_skb);
623 
624 void kfree_skb_list(struct sk_buff *segs)
625 {
626         while (segs) {
627                 struct sk_buff *next = segs->next;
628 
629                 kfree_skb(segs);
630                 segs = next;
631         }
632 }
633 EXPORT_SYMBOL(kfree_skb_list);
634 
635 /**
636  *      skb_tx_error - report an sk_buff xmit error
637  *      @skb: buffer that triggered an error
638  *
639  *      Report xmit error if a device callback is tracking this skb.
640  *      skb must be freed afterwards.
641  */
642 void skb_tx_error(struct sk_buff *skb)
643 {
644         if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) {
645                 struct ubuf_info *uarg;
646 
647                 uarg = skb_shinfo(skb)->destructor_arg;
648                 if (uarg->callback)
649                         uarg->callback(uarg, false);
650                 skb_shinfo(skb)->tx_flags &= ~SKBTX_DEV_ZEROCOPY;
651         }
652 }
653 EXPORT_SYMBOL(skb_tx_error);
654 
655 /**
656  *      consume_skb - free an skbuff
657  *      @skb: buffer to free
658  *
659  *      Drop a ref to the buffer and free it if the usage count has hit zero
660  *      Functions identically to kfree_skb, but kfree_skb assumes that the frame
661  *      is being dropped after a failure and notes that
662  */
663 void consume_skb(struct sk_buff *skb)
664 {
665         if (unlikely(!skb))
666                 return;
667         if (likely(atomic_read(&skb->users) == 1))
668                 smp_rmb();
669         else if (likely(!atomic_dec_and_test(&skb->users)))
670                 return;
671         trace_consume_skb(skb);
672         __kfree_skb(skb);
673 }
674 EXPORT_SYMBOL(consume_skb);
675 
676 static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
677 {
678         new->tstamp             = old->tstamp;
679         new->dev                = old->dev;
680         new->transport_header   = old->transport_header;
681         new->network_header     = old->network_header;
682         new->mac_header         = old->mac_header;
683         new->inner_protocol     = old->inner_protocol;
684         new->inner_transport_header = old->inner_transport_header;
685         new->inner_network_header = old->inner_network_header;
686         new->inner_mac_header = old->inner_mac_header;
687         skb_dst_copy(new, old);
688         skb_copy_hash(new, old);
689         new->ooo_okay           = old->ooo_okay;
690         new->no_fcs             = old->no_fcs;
691         new->encapsulation      = old->encapsulation;
692         new->encap_hdr_csum     = old->encap_hdr_csum;
693         new->csum_valid         = old->csum_valid;
694         new->csum_complete_sw   = old->csum_complete_sw;
695 #ifdef CONFIG_XFRM
696         new->sp                 = secpath_get(old->sp);
697 #endif
698         memcpy(new->cb, old->cb, sizeof(old->cb));
699         new->csum               = old->csum;
700         new->ignore_df          = old->ignore_df;
701         new->pkt_type           = old->pkt_type;
702         new->ip_summed          = old->ip_summed;
703         skb_copy_queue_mapping(new, old);
704         new->priority           = old->priority;
705 #if IS_ENABLED(CONFIG_IP_VS)
706         new->ipvs_property      = old->ipvs_property;
707 #endif
708         new->pfmemalloc         = old->pfmemalloc;
709         new->protocol           = old->protocol;
710         new->mark               = old->mark;
711         new->skb_iif            = old->skb_iif;
712         __nf_copy(new, old);
713 #ifdef CONFIG_NET_SCHED
714         new->tc_index           = old->tc_index;
715 #ifdef CONFIG_NET_CLS_ACT
716         new->tc_verd            = old->tc_verd;
717 #endif
718 #endif
719         new->vlan_proto         = old->vlan_proto;
720         new->vlan_tci           = old->vlan_tci;
721 
722         skb_copy_secmark(new, old);
723 
724 #ifdef CONFIG_NET_RX_BUSY_POLL
725         new->napi_id    = old->napi_id;
726 #endif
727 }
728 
729 /*
730  * You should not add any new code to this function.  Add it to
731  * __copy_skb_header above instead.
732  */
733 static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb)
734 {
735 #define C(x) n->x = skb->x
736 
737         n->next = n->prev = NULL;
738         n->sk = NULL;
739         __copy_skb_header(n, skb);
740 
741         C(len);
742         C(data_len);
743         C(mac_len);
744         n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len;
745         n->cloned = 1;
746         n->nohdr = 0;
747         n->destructor = NULL;
748         C(tail);
749         C(end);
750         C(head);
751         C(head_frag);
752         C(data);
753         C(truesize);
754         atomic_set(&n->users, 1);
755 
756         atomic_inc(&(skb_shinfo(skb)->dataref));
757         skb->cloned = 1;
758 
759         return n;
760 #undef C
761 }
762 
763 /**
764  *      skb_morph       -       morph one skb into another
765  *      @dst: the skb to receive the contents
766  *      @src: the skb to supply the contents
767  *
768  *      This is identical to skb_clone except that the target skb is
769  *      supplied by the user.
770  *
771  *      The target skb is returned upon exit.
772  */
773 struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src)
774 {
775         skb_release_all(dst);
776         return __skb_clone(dst, src);
777 }
778 EXPORT_SYMBOL_GPL(skb_morph);
779 
780 /**
781  *      skb_copy_ubufs  -       copy userspace skb frags buffers to kernel
782  *      @skb: the skb to modify
783  *      @gfp_mask: allocation priority
784  *
785  *      This must be called on SKBTX_DEV_ZEROCOPY skb.
786  *      It will copy all frags into kernel and drop the reference
787  *      to userspace pages.
788  *
789  *      If this function is called from an interrupt gfp_mask() must be
790  *      %GFP_ATOMIC.
791  *
792  *      Returns 0 on success or a negative error code on failure
793  *      to allocate kernel memory to copy to.
794  */
795 int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask)
796 {
797         int i;
798         int num_frags = skb_shinfo(skb)->nr_frags;
799         struct page *page, *head = NULL;
800         struct ubuf_info *uarg = skb_shinfo(skb)->destructor_arg;
801 
802         for (i = 0; i < num_frags; i++) {
803                 u8 *vaddr;
804                 skb_frag_t *f = &skb_shinfo(skb)->frags[i];
805 
806                 page = alloc_page(gfp_mask);
807                 if (!page) {
808                         while (head) {
809                                 struct page *next = (struct page *)page_private(head);
810                                 put_page(head);
811                                 head = next;
812                         }
813                         return -ENOMEM;
814                 }
815                 vaddr = kmap_atomic(skb_frag_page(f));
816                 memcpy(page_address(page),
817                        vaddr + f->page_offset, skb_frag_size(f));
818                 kunmap_atomic(vaddr);
819                 set_page_private(page, (unsigned long)head);
820                 head = page;
821         }
822 
823         /* skb frags release userspace buffers */
824         for (i = 0; i < num_frags; i++)
825                 skb_frag_unref(skb, i);
826 
827         uarg->callback(uarg, false);
828 
829         /* skb frags point to kernel buffers */
830         for (i = num_frags - 1; i >= 0; i--) {
831                 __skb_fill_page_desc(skb, i, head, 0,
832                                      skb_shinfo(skb)->frags[i].size);
833                 head = (struct page *)page_private(head);
834         }
835 
836         skb_shinfo(skb)->tx_flags &= ~SKBTX_DEV_ZEROCOPY;
837         return 0;
838 }
839 EXPORT_SYMBOL_GPL(skb_copy_ubufs);
840 
841 /**
842  *      skb_clone       -       duplicate an sk_buff
843  *      @skb: buffer to clone
844  *      @gfp_mask: allocation priority
845  *
846  *      Duplicate an &sk_buff. The new one is not owned by a socket. Both
847  *      copies share the same packet data but not structure. The new
848  *      buffer has a reference count of 1. If the allocation fails the
849  *      function returns %NULL otherwise the new buffer is returned.
850  *
851  *      If this function is called from an interrupt gfp_mask() must be
852  *      %GFP_ATOMIC.
853  */
854 
855 struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask)
856 {
857         struct sk_buff *n;
858 
859         if (skb_orphan_frags(skb, gfp_mask))
860                 return NULL;
861 
862         n = skb + 1;
863         if (skb->fclone == SKB_FCLONE_ORIG &&
864             n->fclone == SKB_FCLONE_UNAVAILABLE) {
865                 atomic_t *fclone_ref = (atomic_t *) (n + 1);
866                 n->fclone = SKB_FCLONE_CLONE;
867                 atomic_inc(fclone_ref);
868         } else {
869                 if (skb_pfmemalloc(skb))
870                         gfp_mask |= __GFP_MEMALLOC;
871 
872                 n = kmem_cache_alloc(skbuff_head_cache, gfp_mask);
873                 if (!n)
874                         return NULL;
875 
876                 kmemcheck_annotate_bitfield(n, flags1);
877                 kmemcheck_annotate_bitfield(n, flags2);
878                 n->fclone = SKB_FCLONE_UNAVAILABLE;
879         }
880 
881         return __skb_clone(n, skb);
882 }
883 EXPORT_SYMBOL(skb_clone);
884 
885 static void skb_headers_offset_update(struct sk_buff *skb, int off)
886 {
887         /* Only adjust this if it actually is csum_start rather than csum */
888         if (skb->ip_summed == CHECKSUM_PARTIAL)
889                 skb->csum_start += off;
890         /* {transport,network,mac}_header and tail are relative to skb->head */
891         skb->transport_header += off;
892         skb->network_header   += off;
893         if (skb_mac_header_was_set(skb))
894                 skb->mac_header += off;
895         skb->inner_transport_header += off;
896         skb->inner_network_header += off;
897         skb->inner_mac_header += off;
898 }
899 
900 static void copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
901 {
902         __copy_skb_header(new, old);
903 
904         skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size;
905         skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs;
906         skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type;
907 }
908 
909 static inline int skb_alloc_rx_flag(const struct sk_buff *skb)
910 {
911         if (skb_pfmemalloc(skb))
912                 return SKB_ALLOC_RX;
913         return 0;
914 }
915 
916 /**
917  *      skb_copy        -       create private copy of an sk_buff
918  *      @skb: buffer to copy
919  *      @gfp_mask: allocation priority
920  *
921  *      Make a copy of both an &sk_buff and its data. This is used when the
922  *      caller wishes to modify the data and needs a private copy of the
923  *      data to alter. Returns %NULL on failure or the pointer to the buffer
924  *      on success. The returned buffer has a reference count of 1.
925  *
926  *      As by-product this function converts non-linear &sk_buff to linear
927  *      one, so that &sk_buff becomes completely private and caller is allowed
928  *      to modify all the data of returned buffer. This means that this
929  *      function is not recommended for use in circumstances when only
930  *      header is going to be modified. Use pskb_copy() instead.
931  */
932 
933 struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask)
934 {
935         int headerlen = skb_headroom(skb);
936         unsigned int size = skb_end_offset(skb) + skb->data_len;
937         struct sk_buff *n = __alloc_skb(size, gfp_mask,
938                                         skb_alloc_rx_flag(skb), NUMA_NO_NODE);
939 
940         if (!n)
941                 return NULL;
942 
943         /* Set the data pointer */
944         skb_reserve(n, headerlen);
945         /* Set the tail pointer and length */
946         skb_put(n, skb->len);
947 
948         if (skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len))
949                 BUG();
950 
951         copy_skb_header(n, skb);
952         return n;
953 }
954 EXPORT_SYMBOL(skb_copy);
955 
956 /**
957  *      __pskb_copy_fclone      -  create copy of an sk_buff with private head.
958  *      @skb: buffer to copy
959  *      @headroom: headroom of new skb
960  *      @gfp_mask: allocation priority
961  *      @fclone: if true allocate the copy of the skb from the fclone
962  *      cache instead of the head cache; it is recommended to set this
963  *      to true for the cases where the copy will likely be cloned
964  *
965  *      Make a copy of both an &sk_buff and part of its data, located
966  *      in header. Fragmented data remain shared. This is used when
967  *      the caller wishes to modify only header of &sk_buff and needs
968  *      private copy of the header to alter. Returns %NULL on failure
969  *      or the pointer to the buffer on success.
970  *      The returned buffer has a reference count of 1.
971  */
972 
973 struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom,
974                                    gfp_t gfp_mask, bool fclone)
975 {
976         unsigned int size = skb_headlen(skb) + headroom;
977         int flags = skb_alloc_rx_flag(skb) | (fclone ? SKB_ALLOC_FCLONE : 0);
978         struct sk_buff *n = __alloc_skb(size, gfp_mask, flags, NUMA_NO_NODE);
979 
980         if (!n)
981                 goto out;
982 
983         /* Set the data pointer */
984         skb_reserve(n, headroom);
985         /* Set the tail pointer and length */
986         skb_put(n, skb_headlen(skb));
987         /* Copy the bytes */
988         skb_copy_from_linear_data(skb, n->data, n->len);
989 
990         n->truesize += skb->data_len;
991         n->data_len  = skb->data_len;
992         n->len       = skb->len;
993 
994         if (skb_shinfo(skb)->nr_frags) {
995                 int i;
996 
997                 if (skb_orphan_frags(skb, gfp_mask)) {
998                         kfree_skb(n);
999                         n = NULL;
1000                         goto out;
1001                 }
1002                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1003                         skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i];
1004                         skb_frag_ref(skb, i);
1005                 }
1006                 skb_shinfo(n)->nr_frags = i;
1007         }
1008 
1009         if (skb_has_frag_list(skb)) {
1010                 skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list;
1011                 skb_clone_fraglist(n);
1012         }
1013 
1014         copy_skb_header(n, skb);
1015 out:
1016         return n;
1017 }
1018 EXPORT_SYMBOL(__pskb_copy_fclone);
1019 
1020 /**
1021  *      pskb_expand_head - reallocate header of &sk_buff
1022  *      @skb: buffer to reallocate
1023  *      @nhead: room to add at head
1024  *      @ntail: room to add at tail
1025  *      @gfp_mask: allocation priority
1026  *
1027  *      Expands (or creates identical copy, if @nhead and @ntail are zero)
1028  *      header of @skb. &sk_buff itself is not changed. &sk_buff MUST have
1029  *      reference count of 1. Returns zero in the case of success or error,
1030  *      if expansion failed. In the last case, &sk_buff is not changed.
1031  *
1032  *      All the pointers pointing into skb header may change and must be
1033  *      reloaded after call to this function.
1034  */
1035 
1036 int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
1037                      gfp_t gfp_mask)
1038 {
1039         int i;
1040         u8 *data;
1041         int size = nhead + skb_end_offset(skb) + ntail;
1042         long off;
1043 
1044         BUG_ON(nhead < 0);
1045 
1046         if (skb_shared(skb))
1047                 BUG();
1048 
1049         size = SKB_DATA_ALIGN(size);
1050 
1051         if (skb_pfmemalloc(skb))
1052                 gfp_mask |= __GFP_MEMALLOC;
1053         data = kmalloc_reserve(size + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
1054                                gfp_mask, NUMA_NO_NODE, NULL);
1055         if (!data)
1056                 goto nodata;
1057         size = SKB_WITH_OVERHEAD(ksize(data));
1058 
1059         /* Copy only real data... and, alas, header. This should be
1060          * optimized for the cases when header is void.
1061          */
1062         memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head);
1063 
1064         memcpy((struct skb_shared_info *)(data + size),
1065                skb_shinfo(skb),
1066                offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags]));
1067 
1068         /*
1069          * if shinfo is shared we must drop the old head gracefully, but if it
1070          * is not we can just drop the old head and let the existing refcount
1071          * be since all we did is relocate the values
1072          */
1073         if (skb_cloned(skb)) {
1074                 /* copy this zero copy skb frags */
1075                 if (skb_orphan_frags(skb, gfp_mask))
1076                         goto nofrags;
1077                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
1078                         skb_frag_ref(skb, i);
1079 
1080                 if (skb_has_frag_list(skb))
1081                         skb_clone_fraglist(skb);
1082 
1083                 skb_release_data(skb);
1084         } else {
1085                 skb_free_head(skb);
1086         }
1087         off = (data + nhead) - skb->head;
1088 
1089         skb->head     = data;
1090         skb->head_frag = 0;
1091         skb->data    += off;
1092 #ifdef NET_SKBUFF_DATA_USES_OFFSET
1093         skb->end      = size;
1094         off           = nhead;
1095 #else
1096         skb->end      = skb->head + size;
1097 #endif
1098         skb->tail             += off;
1099         skb_headers_offset_update(skb, nhead);
1100         skb->cloned   = 0;
1101         skb->hdr_len  = 0;
1102         skb->nohdr    = 0;
1103         atomic_set(&skb_shinfo(skb)->dataref, 1);
1104         return 0;
1105 
1106 nofrags:
1107         kfree(data);
1108 nodata:
1109         return -ENOMEM;
1110 }
1111 EXPORT_SYMBOL(pskb_expand_head);
1112 
1113 /* Make private copy of skb with writable head and some headroom */
1114 
1115 struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom)
1116 {
1117         struct sk_buff *skb2;
1118         int delta = headroom - skb_headroom(skb);
1119 
1120         if (delta <= 0)
1121                 skb2 = pskb_copy(skb, GFP_ATOMIC);
1122         else {
1123                 skb2 = skb_clone(skb, GFP_ATOMIC);
1124                 if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0,
1125                                              GFP_ATOMIC)) {
1126                         kfree_skb(skb2);
1127                         skb2 = NULL;
1128                 }
1129         }
1130         return skb2;
1131 }
1132 EXPORT_SYMBOL(skb_realloc_headroom);
1133 
1134 /**
1135  *      skb_copy_expand -       copy and expand sk_buff
1136  *      @skb: buffer to copy
1137  *      @newheadroom: new free bytes at head
1138  *      @newtailroom: new free bytes at tail
1139  *      @gfp_mask: allocation priority
1140  *
1141  *      Make a copy of both an &sk_buff and its data and while doing so
1142  *      allocate additional space.
1143  *
1144  *      This is used when the caller wishes to modify the data and needs a
1145  *      private copy of the data to alter as well as more space for new fields.
1146  *      Returns %NULL on failure or the pointer to the buffer
1147  *      on success. The returned buffer has a reference count of 1.
1148  *
1149  *      You must pass %GFP_ATOMIC as the allocation priority if this function
1150  *      is called from an interrupt.
1151  */
1152 struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
1153                                 int newheadroom, int newtailroom,
1154                                 gfp_t gfp_mask)
1155 {
1156         /*
1157          *      Allocate the copy buffer
1158          */
1159         struct sk_buff *n = __alloc_skb(newheadroom + skb->len + newtailroom,
1160                                         gfp_mask, skb_alloc_rx_flag(skb),
1161                                         NUMA_NO_NODE);
1162         int oldheadroom = skb_headroom(skb);
1163         int head_copy_len, head_copy_off;
1164 
1165         if (!n)
1166                 return NULL;
1167 
1168         skb_reserve(n, newheadroom);
1169 
1170         /* Set the tail pointer and length */
1171         skb_put(n, skb->len);
1172 
1173         head_copy_len = oldheadroom;
1174         head_copy_off = 0;
1175         if (newheadroom <= head_copy_len)
1176                 head_copy_len = newheadroom;
1177         else
1178                 head_copy_off = newheadroom - head_copy_len;
1179 
1180         /* Copy the linear header and data. */
1181         if (skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off,
1182                           skb->len + head_copy_len))
1183                 BUG();
1184 
1185         copy_skb_header(n, skb);
1186 
1187         skb_headers_offset_update(n, newheadroom - oldheadroom);
1188 
1189         return n;
1190 }
1191 EXPORT_SYMBOL(skb_copy_expand);
1192 
1193 /**
1194  *      skb_pad                 -       zero pad the tail of an skb
1195  *      @skb: buffer to pad
1196  *      @pad: space to pad
1197  *
1198  *      Ensure that a buffer is followed by a padding area that is zero
1199  *      filled. Used by network drivers which may DMA or transfer data
1200  *      beyond the buffer end onto the wire.
1201  *
1202  *      May return error in out of memory cases. The skb is freed on error.
1203  */
1204 
1205 int skb_pad(struct sk_buff *skb, int pad)
1206 {
1207         int err;
1208         int ntail;
1209 
1210         /* If the skbuff is non linear tailroom is always zero.. */
1211         if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) {
1212                 memset(skb->data+skb->len, 0, pad);
1213                 return 0;
1214         }
1215 
1216         ntail = skb->data_len + pad - (skb->end - skb->tail);
1217         if (likely(skb_cloned(skb) || ntail > 0)) {
1218                 err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC);
1219                 if (unlikely(err))
1220                         goto free_skb;
1221         }
1222 
1223         /* FIXME: The use of this function with non-linear skb's really needs
1224          * to be audited.
1225          */
1226         err = skb_linearize(skb);
1227         if (unlikely(err))
1228                 goto free_skb;
1229 
1230         memset(skb->data + skb->len, 0, pad);
1231         return 0;
1232 
1233 free_skb:
1234         kfree_skb(skb);
1235         return err;
1236 }
1237 EXPORT_SYMBOL(skb_pad);
1238 
1239 /**
1240  *      pskb_put - add data to the tail of a potentially fragmented buffer
1241  *      @skb: start of the buffer to use
1242  *      @tail: tail fragment of the buffer to use
1243  *      @len: amount of data to add
1244  *
1245  *      This function extends the used data area of the potentially
1246  *      fragmented buffer. @tail must be the last fragment of @skb -- or
1247  *      @skb itself. If this would exceed the total buffer size the kernel
1248  *      will panic. A pointer to the first byte of the extra data is
1249  *      returned.
1250  */
1251 
1252 unsigned char *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len)
1253 {
1254         if (tail != skb) {
1255                 skb->data_len += len;
1256                 skb->len += len;
1257         }
1258         return skb_put(tail, len);
1259 }
1260 EXPORT_SYMBOL_GPL(pskb_put);
1261 
1262 /**
1263  *      skb_put - add data to a buffer
1264  *      @skb: buffer to use
1265  *      @len: amount of data to add
1266  *
1267  *      This function extends the used data area of the buffer. If this would
1268  *      exceed the total buffer size the kernel will panic. A pointer to the
1269  *      first byte of the extra data is returned.
1270  */
1271 unsigned char *skb_put(struct sk_buff *skb, unsigned int len)
1272 {
1273         unsigned char *tmp = skb_tail_pointer(skb);
1274         SKB_LINEAR_ASSERT(skb);
1275         skb->tail += len;
1276         skb->len  += len;
1277         if (unlikely(skb->tail > skb->end))
1278                 skb_over_panic(skb, len, __builtin_return_address(0));
1279         return tmp;
1280 }
1281 EXPORT_SYMBOL(skb_put);
1282 
1283 /**
1284  *      skb_push - add data to the start of a buffer
1285  *      @skb: buffer to use
1286  *      @len: amount of data to add
1287  *
1288  *      This function extends the used data area of the buffer at the buffer
1289  *      start. If this would exceed the total buffer headroom the kernel will
1290  *      panic. A pointer to the first byte of the extra data is returned.
1291  */
1292 unsigned char *skb_push(struct sk_buff *skb, unsigned int len)
1293 {
1294         skb->data -= len;
1295         skb->len  += len;
1296         if (unlikely(skb->data<skb->head))
1297                 skb_under_panic(skb, len, __builtin_return_address(0));
1298         return skb->data;
1299 }
1300 EXPORT_SYMBOL(skb_push);
1301 
1302 /**
1303  *      skb_pull - remove data from the start of a buffer
1304  *      @skb: buffer to use
1305  *      @len: amount of data to remove
1306  *
1307  *      This function removes data from the start of a buffer, returning
1308  *      the memory to the headroom. A pointer to the next data in the buffer
1309  *      is returned. Once the data has been pulled future pushes will overwrite
1310  *      the old data.
1311  */
1312 unsigned char *skb_pull(struct sk_buff *skb, unsigned int len)
1313 {
1314         return skb_pull_inline(skb, len);
1315 }
1316 EXPORT_SYMBOL(skb_pull);
1317 
1318 /**
1319  *      skb_trim - remove end from a buffer
1320  *      @skb: buffer to alter
1321  *      @len: new length
1322  *
1323  *      Cut the length of a buffer down by removing data from the tail. If
1324  *      the buffer is already under the length specified it is not modified.
1325  *      The skb must be linear.
1326  */
1327 void skb_trim(struct sk_buff *skb, unsigned int len)
1328 {
1329         if (skb->len > len)
1330                 __skb_trim(skb, len);
1331 }
1332 EXPORT_SYMBOL(skb_trim);
1333 
1334 /* Trims skb to length len. It can change skb pointers.
1335  */
1336 
1337 int ___pskb_trim(struct sk_buff *skb, unsigned int len)
1338 {
1339         struct sk_buff **fragp;
1340         struct sk_buff *frag;
1341         int offset = skb_headlen(skb);
1342         int nfrags = skb_shinfo(skb)->nr_frags;
1343         int i;
1344         int err;
1345 
1346         if (skb_cloned(skb) &&
1347             unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC))))
1348                 return err;
1349 
1350         i = 0;
1351         if (offset >= len)
1352                 goto drop_pages;
1353 
1354         for (; i < nfrags; i++) {
1355                 int end = offset + skb_frag_size(&skb_shinfo(skb)->frags[i]);
1356 
1357                 if (end < len) {
1358                         offset = end;
1359                         continue;
1360                 }
1361 
1362                 skb_frag_size_set(&skb_shinfo(skb)->frags[i++], len - offset);
1363 
1364 drop_pages:
1365                 skb_shinfo(skb)->nr_frags = i;
1366 
1367                 for (; i < nfrags; i++)
1368                         skb_frag_unref(skb, i);
1369 
1370                 if (skb_has_frag_list(skb))
1371                         skb_drop_fraglist(skb);
1372                 goto done;
1373         }
1374 
1375         for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp);
1376              fragp = &frag->next) {
1377                 int end = offset + frag->len;
1378 
1379                 if (skb_shared(frag)) {
1380                         struct sk_buff *nfrag;
1381 
1382                         nfrag = skb_clone(frag, GFP_ATOMIC);
1383                         if (unlikely(!nfrag))
1384                                 return -ENOMEM;
1385 
1386                         nfrag->next = frag->next;
1387                         consume_skb(frag);
1388                         frag = nfrag;
1389                         *fragp = frag;
1390                 }
1391 
1392                 if (end < len) {
1393                         offset = end;
1394                         continue;
1395                 }
1396 
1397                 if (end > len &&
1398                     unlikely((err = pskb_trim(frag, len - offset))))
1399                         return err;
1400 
1401                 if (frag->next)
1402                         skb_drop_list(&frag->next);
1403                 break;
1404         }
1405 
1406 done:
1407         if (len > skb_headlen(skb)) {
1408                 skb->data_len -= skb->len - len;
1409                 skb->len       = len;
1410         } else {
1411                 skb->len       = len;
1412                 skb->data_len  = 0;
1413                 skb_set_tail_pointer(skb, len);
1414         }
1415 
1416         return 0;
1417 }
1418 EXPORT_SYMBOL(___pskb_trim);
1419 
1420 /**
1421  *      __pskb_pull_tail - advance tail of skb header
1422  *      @skb: buffer to reallocate
1423  *      @delta: number of bytes to advance tail
1424  *
1425  *      The function makes a sense only on a fragmented &sk_buff,
1426  *      it expands header moving its tail forward and copying necessary
1427  *      data from fragmented part.
1428  *
1429  *      &sk_buff MUST have reference count of 1.
1430  *
1431  *      Returns %NULL (and &sk_buff does not change) if pull failed
1432  *      or value of new tail of skb in the case of success.
1433  *
1434  *      All the pointers pointing into skb header may change and must be
1435  *      reloaded after call to this function.
1436  */
1437 
1438 /* Moves tail of skb head forward, copying data from fragmented part,
1439  * when it is necessary.
1440  * 1. It may fail due to malloc failure.
1441  * 2. It may change skb pointers.
1442  *
1443  * It is pretty complicated. Luckily, it is called only in exceptional cases.
1444  */
1445 unsigned char *__pskb_pull_tail(struct sk_buff *skb, int delta)
1446 {
1447         /* If skb has not enough free space at tail, get new one
1448          * plus 128 bytes for future expansions. If we have enough
1449          * room at tail, reallocate without expansion only if skb is cloned.
1450          */
1451         int i, k, eat = (skb->tail + delta) - skb->end;
1452 
1453         if (eat > 0 || skb_cloned(skb)) {
1454                 if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0,
1455                                      GFP_ATOMIC))
1456                         return NULL;
1457         }
1458 
1459         if (skb_copy_bits(skb, skb_headlen(skb), skb_tail_pointer(skb), delta))
1460                 BUG();
1461 
1462         /* Optimization: no fragments, no reasons to preestimate
1463          * size of pulled pages. Superb.
1464          */
1465         if (!skb_has_frag_list(skb))
1466                 goto pull_pages;
1467 
1468         /* Estimate size of pulled pages. */
1469         eat = delta;
1470         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1471                 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
1472 
1473                 if (size >= eat)
1474                         goto pull_pages;
1475                 eat -= size;
1476         }
1477 
1478         /* If we need update frag list, we are in troubles.
1479          * Certainly, it possible to add an offset to skb data,
1480          * but taking into account that pulling is expected to
1481          * be very rare operation, it is worth to fight against
1482          * further bloating skb head and crucify ourselves here instead.
1483          * Pure masohism, indeed. 8)8)
1484          */
1485         if (eat) {
1486                 struct sk_buff *list = skb_shinfo(skb)->frag_list;
1487                 struct sk_buff *clone = NULL;
1488                 struct sk_buff *insp = NULL;
1489 
1490                 do {
1491                         BUG_ON(!list);
1492 
1493                         if (list->len <= eat) {
1494                                 /* Eaten as whole. */
1495                                 eat -= list->len;
1496                                 list = list->next;
1497                                 insp = list;
1498                         } else {
1499                                 /* Eaten partially. */
1500 
1501                                 if (skb_shared(list)) {
1502                                         /* Sucks! We need to fork list. :-( */
1503                                         clone = skb_clone(list, GFP_ATOMIC);
1504                                         if (!clone)
1505                                                 return NULL;
1506                                         insp = list->next;
1507                                         list = clone;
1508                                 } else {
1509                                         /* This may be pulled without
1510                                          * problems. */
1511                                         insp = list;
1512                                 }
1513                                 if (!pskb_pull(list, eat)) {
1514                                         kfree_skb(clone);
1515                                         return NULL;
1516                                 }
1517                                 break;
1518                         }
1519                 } while (eat);
1520 
1521                 /* Free pulled out fragments. */
1522                 while ((list = skb_shinfo(skb)->frag_list) != insp) {
1523                         skb_shinfo(skb)->frag_list = list->next;
1524                         kfree_skb(list);
1525                 }
1526                 /* And insert new clone at head. */
1527                 if (clone) {
1528                         clone->next = list;
1529                         skb_shinfo(skb)->frag_list = clone;
1530                 }
1531         }
1532         /* Success! Now we may commit changes to skb data. */
1533 
1534 pull_pages:
1535         eat = delta;
1536         k = 0;
1537         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1538                 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
1539 
1540                 if (size <= eat) {
1541                         skb_frag_unref(skb, i);
1542                         eat -= size;
1543                 } else {
1544                         skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i];
1545                         if (eat) {
1546                                 skb_shinfo(skb)->frags[k].page_offset += eat;
1547                                 skb_frag_size_sub(&skb_shinfo(skb)->frags[k], eat);
1548                                 eat = 0;
1549                         }
1550                         k++;
1551                 }
1552         }
1553         skb_shinfo(skb)->nr_frags = k;
1554 
1555         skb->tail     += delta;
1556         skb->data_len -= delta;
1557 
1558         return skb_tail_pointer(skb);
1559 }
1560 EXPORT_SYMBOL(__pskb_pull_tail);
1561 
1562 /**
1563  *      skb_copy_bits - copy bits from skb to kernel buffer
1564  *      @skb: source skb
1565  *      @offset: offset in source
1566  *      @to: destination buffer
1567  *      @len: number of bytes to copy
1568  *
1569  *      Copy the specified number of bytes from the source skb to the
1570  *      destination buffer.
1571  *
1572  *      CAUTION ! :
1573  *              If its prototype is ever changed,
1574  *              check arch/{*}/net/{*}.S files,
1575  *              since it is called from BPF assembly code.
1576  */
1577 int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len)
1578 {
1579         int start = skb_headlen(skb);
1580         struct sk_buff *frag_iter;
1581         int i, copy;
1582 
1583         if (offset > (int)skb->len - len)
1584                 goto fault;
1585 
1586         /* Copy header. */
1587         if ((copy = start - offset) > 0) {
1588                 if (copy > len)
1589                         copy = len;
1590                 skb_copy_from_linear_data_offset(skb, offset, to, copy);
1591                 if ((len -= copy) == 0)
1592                         return 0;
1593                 offset += copy;
1594                 to     += copy;
1595         }
1596 
1597         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1598                 int end;
1599                 skb_frag_t *f = &skb_shinfo(skb)->frags[i];
1600 
1601                 WARN_ON(start > offset + len);
1602 
1603                 end = start + skb_frag_size(f);
1604                 if ((copy = end - offset) > 0) {
1605                         u8 *vaddr;
1606 
1607                         if (copy > len)
1608                                 copy = len;
1609 
1610                         vaddr = kmap_atomic(skb_frag_page(f));
1611                         memcpy(to,
1612                                vaddr + f->page_offset + offset - start,
1613                                copy);
1614                         kunmap_atomic(vaddr);
1615 
1616                         if ((len -= copy) == 0)
1617                                 return 0;
1618                         offset += copy;
1619                         to     += copy;
1620                 }
1621                 start = end;
1622         }
1623 
1624         skb_walk_frags(skb, frag_iter) {
1625                 int end;
1626 
1627                 WARN_ON(start > offset + len);
1628 
1629                 end = start + frag_iter->len;
1630                 if ((copy = end - offset) > 0) {
1631                         if (copy > len)
1632                                 copy = len;
1633                         if (skb_copy_bits(frag_iter, offset - start, to, copy))
1634                                 goto fault;
1635                         if ((len -= copy) == 0)
1636                                 return 0;
1637                         offset += copy;
1638                         to     += copy;
1639                 }
1640                 start = end;
1641         }
1642 
1643         if (!len)
1644                 return 0;
1645 
1646 fault:
1647         return -EFAULT;
1648 }
1649 EXPORT_SYMBOL(skb_copy_bits);
1650 
1651 /*
1652  * Callback from splice_to_pipe(), if we need to release some pages
1653  * at the end of the spd in case we error'ed out in filling the pipe.
1654  */
1655 static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i)
1656 {
1657         put_page(spd->pages[i]);
1658 }
1659 
1660 static struct page *linear_to_page(struct page *page, unsigned int *len,
1661                                    unsigned int *offset,
1662                                    struct sock *sk)
1663 {
1664         struct page_frag *pfrag = sk_page_frag(sk);
1665 
1666         if (!sk_page_frag_refill(sk, pfrag))
1667                 return NULL;
1668 
1669         *len = min_t(unsigned int, *len, pfrag->size - pfrag->offset);
1670 
1671         memcpy(page_address(pfrag->page) + pfrag->offset,
1672                page_address(page) + *offset, *len);
1673         *offset = pfrag->offset;
1674         pfrag->offset += *len;
1675 
1676         return pfrag->page;
1677 }
1678 
1679 static bool spd_can_coalesce(const struct splice_pipe_desc *spd,
1680                              struct page *page,
1681                              unsigned int offset)
1682 {
1683         return  spd->nr_pages &&
1684                 spd->pages[spd->nr_pages - 1] == page &&
1685                 (spd->partial[spd->nr_pages - 1].offset +
1686                  spd->partial[spd->nr_pages - 1].len == offset);
1687 }
1688 
1689 /*
1690  * Fill page/offset/length into spd, if it can hold more pages.
1691  */
1692 static bool spd_fill_page(struct splice_pipe_desc *spd,
1693                           struct pipe_inode_info *pipe, struct page *page,
1694                           unsigned int *len, unsigned int offset,
1695                           bool linear,
1696                           struct sock *sk)
1697 {
1698         if (unlikely(spd->nr_pages == MAX_SKB_FRAGS))
1699                 return true;
1700 
1701         if (linear) {
1702                 page = linear_to_page(page, len, &offset, sk);
1703                 if (!page)
1704                         return true;
1705         }
1706         if (spd_can_coalesce(spd, page, offset)) {
1707                 spd->partial[spd->nr_pages - 1].len += *len;
1708                 return false;
1709         }
1710         get_page(page);
1711         spd->pages[spd->nr_pages] = page;
1712         spd->partial[spd->nr_pages].len = *len;
1713         spd->partial[spd->nr_pages].offset = offset;
1714         spd->nr_pages++;
1715 
1716         return false;
1717 }
1718 
1719 static bool __splice_segment(struct page *page, unsigned int poff,
1720                              unsigned int plen, unsigned int *off,
1721                              unsigned int *len,
1722                              struct splice_pipe_desc *spd, bool linear,
1723                              struct sock *sk,
1724                              struct pipe_inode_info *pipe)
1725 {
1726         if (!*len)
1727                 return true;
1728 
1729         /* skip this segment if already processed */
1730         if (*off >= plen) {
1731                 *off -= plen;
1732                 return false;
1733         }
1734 
1735         /* ignore any bits we already processed */
1736         poff += *off;
1737         plen -= *off;
1738         *off = 0;
1739 
1740         do {
1741                 unsigned int flen = min(*len, plen);
1742 
1743                 if (spd_fill_page(spd, pipe, page, &flen, poff,
1744                                   linear, sk))
1745                         return true;
1746                 poff += flen;
1747                 plen -= flen;
1748                 *len -= flen;
1749         } while (*len && plen);
1750 
1751         return false;
1752 }
1753 
1754 /*
1755  * Map linear and fragment data from the skb to spd. It reports true if the
1756  * pipe is full or if we already spliced the requested length.
1757  */
1758 static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe,
1759                               unsigned int *offset, unsigned int *len,
1760                               struct splice_pipe_desc *spd, struct sock *sk)
1761 {
1762         int seg;
1763 
1764         /* map the linear part :
1765          * If skb->head_frag is set, this 'linear' part is backed by a
1766          * fragment, and if the head is not shared with any clones then
1767          * we can avoid a copy since we own the head portion of this page.
1768          */
1769         if (__splice_segment(virt_to_page(skb->data),
1770                              (unsigned long) skb->data & (PAGE_SIZE - 1),
1771                              skb_headlen(skb),
1772                              offset, len, spd,
1773                              skb_head_is_locked(skb),
1774                              sk, pipe))
1775                 return true;
1776 
1777         /*
1778          * then map the fragments
1779          */
1780         for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) {
1781                 const skb_frag_t *f = &skb_shinfo(skb)->frags[seg];
1782 
1783                 if (__splice_segment(skb_frag_page(f),
1784                                      f->page_offset, skb_frag_size(f),
1785                                      offset, len, spd, false, sk, pipe))
1786                         return true;
1787         }
1788 
1789         return false;
1790 }
1791 
1792 /*
1793  * Map data from the skb to a pipe. Should handle both the linear part,
1794  * the fragments, and the frag list. It does NOT handle frag lists within
1795  * the frag list, if such a thing exists. We'd probably need to recurse to
1796  * handle that cleanly.
1797  */
1798 int skb_splice_bits(struct sk_buff *skb, unsigned int offset,
1799                     struct pipe_inode_info *pipe, unsigned int tlen,
1800                     unsigned int flags)
1801 {
1802         struct partial_page partial[MAX_SKB_FRAGS];
1803         struct page *pages[MAX_SKB_FRAGS];
1804         struct splice_pipe_desc spd = {
1805                 .pages = pages,
1806                 .partial = partial,
1807                 .nr_pages_max = MAX_SKB_FRAGS,
1808                 .flags = flags,
1809                 .ops = &nosteal_pipe_buf_ops,
1810                 .spd_release = sock_spd_release,
1811         };
1812         struct sk_buff *frag_iter;
1813         struct sock *sk = skb->sk;
1814         int ret = 0;
1815 
1816         /*
1817          * __skb_splice_bits() only fails if the output has no room left,
1818          * so no point in going over the frag_list for the error case.
1819          */
1820         if (__skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk))
1821                 goto done;
1822         else if (!tlen)
1823                 goto done;
1824 
1825         /*
1826          * now see if we have a frag_list to map
1827          */
1828         skb_walk_frags(skb, frag_iter) {
1829                 if (!tlen)
1830                         break;
1831                 if (__skb_splice_bits(frag_iter, pipe, &offset, &tlen, &spd, sk))
1832                         break;
1833         }
1834 
1835 done:
1836         if (spd.nr_pages) {
1837                 /*
1838                  * Drop the socket lock, otherwise we have reverse
1839                  * locking dependencies between sk_lock and i_mutex
1840                  * here as compared to sendfile(). We enter here
1841                  * with the socket lock held, and splice_to_pipe() will
1842                  * grab the pipe inode lock. For sendfile() emulation,
1843                  * we call into ->sendpage() with the i_mutex lock held
1844                  * and networking will grab the socket lock.
1845                  */
1846                 release_sock(sk);
1847                 ret = splice_to_pipe(pipe, &spd);
1848                 lock_sock(sk);
1849         }
1850 
1851         return ret;
1852 }
1853 
1854 /**
1855  *      skb_store_bits - store bits from kernel buffer to skb
1856  *      @skb: destination buffer
1857  *      @offset: offset in destination
1858  *      @from: source buffer
1859  *      @len: number of bytes to copy
1860  *
1861  *      Copy the specified number of bytes from the source buffer to the
1862  *      destination skb.  This function handles all the messy bits of
1863  *      traversing fragment lists and such.
1864  */
1865 
1866 int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len)
1867 {
1868         int start = skb_headlen(skb);
1869         struct sk_buff *frag_iter;
1870         int i, copy;
1871 
1872         if (offset > (int)skb->len - len)
1873                 goto fault;
1874 
1875         if ((copy = start - offset) > 0) {
1876                 if (copy > len)
1877                         copy = len;
1878                 skb_copy_to_linear_data_offset(skb, offset, from, copy);
1879                 if ((len -= copy) == 0)
1880                         return 0;
1881                 offset += copy;
1882                 from += copy;
1883         }
1884 
1885         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1886                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1887                 int end;
1888 
1889                 WARN_ON(start > offset + len);
1890 
1891                 end = start + skb_frag_size(frag);
1892                 if ((copy = end - offset) > 0) {
1893                         u8 *vaddr;
1894 
1895                         if (copy > len)
1896                                 copy = len;
1897 
1898                         vaddr = kmap_atomic(skb_frag_page(frag));
1899                         memcpy(vaddr + frag->page_offset + offset - start,
1900                                from, copy);
1901                         kunmap_atomic(vaddr);
1902 
1903                         if ((len -= copy) == 0)
1904                                 return 0;
1905                         offset += copy;
1906                         from += copy;
1907                 }
1908                 start = end;
1909         }
1910 
1911         skb_walk_frags(skb, frag_iter) {
1912                 int end;
1913 
1914                 WARN_ON(start > offset + len);
1915 
1916                 end = start + frag_iter->len;
1917                 if ((copy = end - offset) > 0) {
1918                         if (copy > len)
1919                                 copy = len;
1920                         if (skb_store_bits(frag_iter, offset - start,
1921                                            from, copy))
1922                                 goto fault;
1923                         if ((len -= copy) == 0)
1924                                 return 0;
1925                         offset += copy;
1926                         from += copy;
1927                 }
1928                 start = end;
1929         }
1930         if (!len)
1931                 return 0;
1932 
1933 fault:
1934         return -EFAULT;
1935 }
1936 EXPORT_SYMBOL(skb_store_bits);
1937 
1938 /* Checksum skb data. */
1939 __wsum __skb_checksum(const struct sk_buff *skb, int offset, int len,
1940                       __wsum csum, const struct skb_checksum_ops *ops)
1941 {
1942         int start = skb_headlen(skb);
1943         int i, copy = start - offset;
1944         struct sk_buff *frag_iter;
1945         int pos = 0;
1946 
1947         /* Checksum header. */
1948         if (copy > 0) {
1949                 if (copy > len)
1950                         copy = len;
1951                 csum = ops->update(skb->data + offset, copy, csum);
1952                 if ((len -= copy) == 0)
1953                         return csum;
1954                 offset += copy;
1955                 pos     = copy;
1956         }
1957 
1958         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1959                 int end;
1960                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1961 
1962                 WARN_ON(start > offset + len);
1963 
1964                 end = start + skb_frag_size(frag);
1965                 if ((copy = end - offset) > 0) {
1966                         __wsum csum2;
1967                         u8 *vaddr;
1968 
1969                         if (copy > len)
1970                                 copy = len;
1971                         vaddr = kmap_atomic(skb_frag_page(frag));
1972                         csum2 = ops->update(vaddr + frag->page_offset +
1973                                             offset - start, copy, 0);
1974                         kunmap_atomic(vaddr);
1975                         csum = ops->combine(csum, csum2, pos, copy);
1976                         if (!(len -= copy))
1977                                 return csum;
1978                         offset += copy;
1979                         pos    += copy;
1980                 }
1981                 start = end;
1982         }
1983 
1984         skb_walk_frags(skb, frag_iter) {
1985                 int end;
1986 
1987                 WARN_ON(start > offset + len);
1988 
1989                 end = start + frag_iter->len;
1990                 if ((copy = end - offset) > 0) {
1991                         __wsum csum2;
1992                         if (copy > len)
1993                                 copy = len;
1994                         csum2 = __skb_checksum(frag_iter, offset - start,
1995                                                copy, 0, ops);
1996                         csum = ops->combine(csum, csum2, pos, copy);
1997                         if ((len -= copy) == 0)
1998                                 return csum;
1999                         offset += copy;
2000                         pos    += copy;
2001                 }
2002                 start = end;
2003         }
2004         BUG_ON(len);
2005 
2006         return csum;
2007 }
2008 EXPORT_SYMBOL(__skb_checksum);
2009 
2010 __wsum skb_checksum(const struct sk_buff *skb, int offset,
2011                     int len, __wsum csum)
2012 {
2013         const struct skb_checksum_ops ops = {
2014                 .update  = csum_partial_ext,
2015                 .combine = csum_block_add_ext,
2016         };
2017 
2018         return __skb_checksum(skb, offset, len, csum, &ops);
2019 }
2020 EXPORT_SYMBOL(skb_checksum);
2021 
2022 /* Both of above in one bottle. */
2023 
2024 __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset,
2025                                     u8 *to, int len, __wsum csum)
2026 {
2027         int start = skb_headlen(skb);
2028         int i, copy = start - offset;
2029         struct sk_buff *frag_iter;
2030         int pos = 0;
2031 
2032         /* Copy header. */
2033         if (copy > 0) {
2034                 if (copy > len)
2035                         copy = len;
2036                 csum = csum_partial_copy_nocheck(skb->data + offset, to,
2037                                                  copy, csum);
2038                 if ((len -= copy) == 0)
2039                         return csum;
2040                 offset += copy;
2041                 to     += copy;
2042                 pos     = copy;
2043         }
2044 
2045         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2046                 int end;
2047 
2048                 WARN_ON(start > offset + len);
2049 
2050                 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
2051                 if ((copy = end - offset) > 0) {
2052                         __wsum csum2;
2053                         u8 *vaddr;
2054                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2055 
2056                         if (copy > len)
2057                                 copy = len;
2058                         vaddr = kmap_atomic(skb_frag_page(frag));
2059                         csum2 = csum_partial_copy_nocheck(vaddr +
2060                                                           frag->page_offset +
2061                                                           offset - start, to,
2062                                                           copy, 0);
2063                         kunmap_atomic(vaddr);
2064                         csum = csum_block_add(csum, csum2, pos);
2065                         if (!(len -= copy))
2066                                 return csum;
2067                         offset += copy;
2068                         to     += copy;
2069                         pos    += copy;
2070                 }
2071                 start = end;
2072         }
2073 
2074         skb_walk_frags(skb, frag_iter) {
2075                 __wsum csum2;
2076                 int end;
2077 
2078                 WARN_ON(start > offset + len);
2079 
2080                 end = start + frag_iter->len;
2081                 if ((copy = end - offset) > 0) {
2082                         if (copy > len)
2083                                 copy = len;
2084                         csum2 = skb_copy_and_csum_bits(frag_iter,
2085                                                        offset - start,
2086                                                        to, copy, 0);
2087                         csum = csum_block_add(csum, csum2, pos);
2088                         if ((len -= copy) == 0)
2089                                 return csum;
2090                         offset += copy;
2091                         to     += copy;
2092                         pos    += copy;
2093                 }
2094                 start = end;
2095         }
2096         BUG_ON(len);
2097         return csum;
2098 }
2099 EXPORT_SYMBOL(skb_copy_and_csum_bits);
2100 
2101  /**
2102  *      skb_zerocopy_headlen - Calculate headroom needed for skb_zerocopy()
2103  *      @from: source buffer
2104  *
2105  *      Calculates the amount of linear headroom needed in the 'to' skb passed
2106  *      into skb_zerocopy().
2107  */
2108 unsigned int
2109 skb_zerocopy_headlen(const struct sk_buff *from)
2110 {
2111         unsigned int hlen = 0;
2112 
2113         if (!from->head_frag ||
2114             skb_headlen(from) < L1_CACHE_BYTES ||
2115             skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS)
2116                 hlen = skb_headlen(from);
2117 
2118         if (skb_has_frag_list(from))
2119                 hlen = from->len;
2120 
2121         return hlen;
2122 }
2123 EXPORT_SYMBOL_GPL(skb_zerocopy_headlen);
2124 
2125 /**
2126  *      skb_zerocopy - Zero copy skb to skb
2127  *      @to: destination buffer
2128  *      @from: source buffer
2129  *      @len: number of bytes to copy from source buffer
2130  *      @hlen: size of linear headroom in destination buffer
2131  *
2132  *      Copies up to `len` bytes from `from` to `to` by creating references
2133  *      to the frags in the source buffer.
2134  *
2135  *      The `hlen` as calculated by skb_zerocopy_headlen() specifies the
2136  *      headroom in the `to` buffer.
2137  *
2138  *      Return value:
2139  *      0: everything is OK
2140  *      -ENOMEM: couldn't orphan frags of @from due to lack of memory
2141  *      -EFAULT: skb_copy_bits() found some problem with skb geometry
2142  */
2143 int
2144 skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen)
2145 {
2146         int i, j = 0;
2147         int plen = 0; /* length of skb->head fragment */
2148         int ret;
2149         struct page *page;
2150         unsigned int offset;
2151 
2152         BUG_ON(!from->head_frag && !hlen);
2153 
2154         /* dont bother with small payloads */
2155         if (len <= skb_tailroom(to))
2156                 return skb_copy_bits(from, 0, skb_put(to, len), len);
2157 
2158         if (hlen) {
2159                 ret = skb_copy_bits(from, 0, skb_put(to, hlen), hlen);
2160                 if (unlikely(ret))
2161                         return ret;
2162                 len -= hlen;
2163         } else {
2164                 plen = min_t(int, skb_headlen(from), len);
2165                 if (plen) {
2166                         page = virt_to_head_page(from->head);
2167                         offset = from->data - (unsigned char *)page_address(page);
2168                         __skb_fill_page_desc(to, 0, page, offset, plen);
2169                         get_page(page);
2170                         j = 1;
2171                         len -= plen;
2172                 }
2173         }
2174 
2175         to->truesize += len + plen;
2176         to->len += len + plen;
2177         to->data_len += len + plen;
2178 
2179         if (unlikely(skb_orphan_frags(from, GFP_ATOMIC))) {
2180                 skb_tx_error(from);
2181                 return -ENOMEM;
2182         }
2183 
2184         for (i = 0; i < skb_shinfo(from)->nr_frags; i++) {
2185                 if (!len)
2186                         break;
2187                 skb_shinfo(to)->frags[j] = skb_shinfo(from)->frags[i];
2188                 skb_shinfo(to)->frags[j].size = min_t(int, skb_shinfo(to)->frags[j].size, len);
2189                 len -= skb_shinfo(to)->frags[j].size;
2190                 skb_frag_ref(to, j);
2191                 j++;
2192         }
2193         skb_shinfo(to)->nr_frags = j;
2194 
2195         return 0;
2196 }
2197 EXPORT_SYMBOL_GPL(skb_zerocopy);
2198 
2199 void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to)
2200 {
2201         __wsum csum;
2202         long csstart;
2203 
2204         if (skb->ip_summed == CHECKSUM_PARTIAL)
2205                 csstart = skb_checksum_start_offset(skb);
2206         else
2207                 csstart = skb_headlen(skb);
2208 
2209         BUG_ON(csstart > skb_headlen(skb));
2210 
2211         skb_copy_from_linear_data(skb, to, csstart);
2212 
2213         csum = 0;
2214         if (csstart != skb->len)
2215                 csum = skb_copy_and_csum_bits(skb, csstart, to + csstart,
2216                                               skb->len - csstart, 0);
2217 
2218         if (skb->ip_summed == CHECKSUM_PARTIAL) {
2219                 long csstuff = csstart + skb->csum_offset;
2220 
2221                 *((__sum16 *)(to + csstuff)) = csum_fold(csum);
2222         }
2223 }
2224 EXPORT_SYMBOL(skb_copy_and_csum_dev);
2225 
2226 /**
2227  *      skb_dequeue - remove from the head of the queue
2228  *      @list: list to dequeue from
2229  *
2230  *      Remove the head of the list. The list lock is taken so the function
2231  *      may be used safely with other locking list functions. The head item is
2232  *      returned or %NULL if the list is empty.
2233  */
2234 
2235 struct sk_buff *skb_dequeue(struct sk_buff_head *list)
2236 {
2237         unsigned long flags;
2238         struct sk_buff *result;
2239 
2240         spin_lock_irqsave(&list->lock, flags);
2241         result = __skb_dequeue(list);
2242         spin_unlock_irqrestore(&list->lock, flags);
2243         return result;
2244 }
2245 EXPORT_SYMBOL(skb_dequeue);
2246 
2247 /**
2248  *      skb_dequeue_tail - remove from the tail of the queue
2249  *      @list: list to dequeue from
2250  *
2251  *      Remove the tail of the list. The list lock is taken so the function
2252  *      may be used safely with other locking list functions. The tail item is
2253  *      returned or %NULL if the list is empty.
2254  */
2255 struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list)
2256 {
2257         unsigned long flags;
2258         struct sk_buff *result;
2259 
2260         spin_lock_irqsave(&list->lock, flags);
2261         result = __skb_dequeue_tail(list);
2262         spin_unlock_irqrestore(&list->lock, flags);
2263         return result;
2264 }
2265 EXPORT_SYMBOL(skb_dequeue_tail);
2266 
2267 /**
2268  *      skb_queue_purge - empty a list
2269  *      @list: list to empty
2270  *
2271  *      Delete all buffers on an &sk_buff list. Each buffer is removed from
2272  *      the list and one reference dropped. This function takes the list
2273  *      lock and is atomic with respect to other list locking functions.
2274  */
2275 void skb_queue_purge(struct sk_buff_head *list)
2276 {
2277         struct sk_buff *skb;
2278         while ((skb = skb_dequeue(list)) != NULL)
2279                 kfree_skb(skb);
2280 }
2281 EXPORT_SYMBOL(skb_queue_purge);
2282 
2283 /**
2284  *      skb_queue_head - queue a buffer at the list head
2285  *      @list: list to use
2286  *      @newsk: buffer to queue
2287  *
2288  *      Queue a buffer at the start of the list. This function takes the
2289  *      list lock and can be used safely with other locking &sk_buff functions
2290  *      safely.
2291  *
2292  *      A buffer cannot be placed on two lists at the same time.
2293  */
2294 void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk)
2295 {
2296         unsigned long flags;
2297 
2298         spin_lock_irqsave(&list->lock, flags);
2299         __skb_queue_head(list, newsk);
2300         spin_unlock_irqrestore(&list->lock, flags);
2301 }
2302 EXPORT_SYMBOL(skb_queue_head);
2303 
2304 /**
2305  *      skb_queue_tail - queue a buffer at the list tail
2306  *      @list: list to use
2307  *      @newsk: buffer to queue
2308  *
2309  *      Queue a buffer at the tail of the list. This function takes the
2310  *      list lock and can be used safely with other locking &sk_buff functions
2311  *      safely.
2312  *
2313  *      A buffer cannot be placed on two lists at the same time.
2314  */
2315 void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk)
2316 {
2317         unsigned long flags;
2318 
2319         spin_lock_irqsave(&list->lock, flags);
2320         __skb_queue_tail(list, newsk);
2321         spin_unlock_irqrestore(&list->lock, flags);
2322 }
2323 EXPORT_SYMBOL(skb_queue_tail);
2324 
2325 /**
2326  *      skb_unlink      -       remove a buffer from a list
2327  *      @skb: buffer to remove
2328  *      @list: list to use
2329  *
2330  *      Remove a packet from a list. The list locks are taken and this
2331  *      function is atomic with respect to other list locked calls
2332  *
2333  *      You must know what list the SKB is on.
2334  */
2335 void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
2336 {
2337         unsigned long flags;
2338 
2339         spin_lock_irqsave(&list->lock, flags);
2340         __skb_unlink(skb, list);
2341         spin_unlock_irqrestore(&list->lock, flags);
2342 }
2343 EXPORT_SYMBOL(skb_unlink);
2344 
2345 /**
2346  *      skb_append      -       append a buffer
2347  *      @old: buffer to insert after
2348  *      @newsk: buffer to insert
2349  *      @list: list to use
2350  *
2351  *      Place a packet after a given packet in a list. The list locks are taken
2352  *      and this function is atomic with respect to other list locked calls.
2353  *      A buffer cannot be placed on two lists at the same time.
2354  */
2355 void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
2356 {
2357         unsigned long flags;
2358 
2359         spin_lock_irqsave(&list->lock, flags);
2360         __skb_queue_after(list, old, newsk);
2361         spin_unlock_irqrestore(&list->lock, flags);
2362 }
2363 EXPORT_SYMBOL(skb_append);
2364 
2365 /**
2366  *      skb_insert      -       insert a buffer
2367  *      @old: buffer to insert before
2368  *      @newsk: buffer to insert
2369  *      @list: list to use
2370  *
2371  *      Place a packet before a given packet in a list. The list locks are
2372  *      taken and this function is atomic with respect to other list locked
2373  *      calls.
2374  *
2375  *      A buffer cannot be placed on two lists at the same time.
2376  */
2377 void skb_insert(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
2378 {
2379         unsigned long flags;
2380 
2381         spin_lock_irqsave(&list->lock, flags);
2382         __skb_insert(newsk, old->prev, old, list);
2383         spin_unlock_irqrestore(&list->lock, flags);
2384 }
2385 EXPORT_SYMBOL(skb_insert);
2386 
2387 static inline void skb_split_inside_header(struct sk_buff *skb,
2388                                            struct sk_buff* skb1,
2389                                            const u32 len, const int pos)
2390 {
2391         int i;
2392 
2393         skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len),
2394                                          pos - len);
2395         /* And move data appendix as is. */
2396         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
2397                 skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i];
2398 
2399         skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags;
2400         skb_shinfo(skb)->nr_frags  = 0;
2401         skb1->data_len             = skb->data_len;
2402         skb1->len                  += skb1->data_len;
2403         skb->data_len              = 0;
2404         skb->len                   = len;
2405         skb_set_tail_pointer(skb, len);
2406 }
2407 
2408 static inline void skb_split_no_header(struct sk_buff *skb,
2409                                        struct sk_buff* skb1,
2410                                        const u32 len, int pos)
2411 {
2412         int i, k = 0;
2413         const int nfrags = skb_shinfo(skb)->nr_frags;
2414 
2415         skb_shinfo(skb)->nr_frags = 0;
2416         skb1->len                 = skb1->data_len = skb->len - len;
2417         skb->len                  = len;
2418         skb->data_len             = len - pos;
2419 
2420         for (i = 0; i < nfrags; i++) {
2421                 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
2422 
2423                 if (pos + size > len) {
2424                         skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i];
2425 
2426                         if (pos < len) {
2427                                 /* Split frag.
2428                                  * We have two variants in this case:
2429                                  * 1. Move all the frag to the second
2430                                  *    part, if it is possible. F.e.
2431                                  *    this approach is mandatory for TUX,
2432                                  *    where splitting is expensive.
2433                                  * 2. Split is accurately. We make this.
2434                                  */
2435                                 skb_frag_ref(skb, i);
2436                                 skb_shinfo(skb1)->frags[0].page_offset += len - pos;
2437                                 skb_frag_size_sub(&skb_shinfo(skb1)->frags[0], len - pos);
2438                                 skb_frag_size_set(&skb_shinfo(skb)->frags[i], len - pos);
2439                                 skb_shinfo(skb)->nr_frags++;
2440                         }
2441                         k++;
2442                 } else
2443                         skb_shinfo(skb)->nr_frags++;
2444                 pos += size;
2445         }
2446         skb_shinfo(skb1)->nr_frags = k;
2447 }
2448 
2449 /**
2450  * skb_split - Split fragmented skb to two parts at length len.
2451  * @skb: the buffer to split
2452  * @skb1: the buffer to receive the second part
2453  * @len: new length for skb
2454  */
2455 void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len)
2456 {
2457         int pos = skb_headlen(skb);
2458 
2459         skb_shinfo(skb1)->tx_flags = skb_shinfo(skb)->tx_flags & SKBTX_SHARED_FRAG;
2460         if (len < pos)  /* Split line is inside header. */
2461                 skb_split_inside_header(skb, skb1, len, pos);
2462         else            /* Second chunk has no header, nothing to copy. */
2463                 skb_split_no_header(skb, skb1, len, pos);
2464 }
2465 EXPORT_SYMBOL(skb_split);
2466 
2467 /* Shifting from/to a cloned skb is a no-go.
2468  *
2469  * Caller cannot keep skb_shinfo related pointers past calling here!
2470  */
2471 static int skb_prepare_for_shift(struct sk_buff *skb)
2472 {
2473         return skb_cloned(skb) && pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2474 }
2475 
2476 /**
2477  * skb_shift - Shifts paged data partially from skb to another
2478  * @tgt: buffer into which tail data gets added
2479  * @skb: buffer from which the paged data comes from
2480  * @shiftlen: shift up to this many bytes
2481  *
2482  * Attempts to shift up to shiftlen worth of bytes, which may be less than
2483  * the length of the skb, from skb to tgt. Returns number bytes shifted.
2484  * It's up to caller to free skb if everything was shifted.
2485  *
2486  * If @tgt runs out of frags, the whole operation is aborted.
2487  *
2488  * Skb cannot include anything else but paged data while tgt is allowed
2489  * to have non-paged data as well.
2490  *
2491  * TODO: full sized shift could be optimized but that would need
2492  * specialized skb free'er to handle frags without up-to-date nr_frags.
2493  */
2494 int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen)
2495 {
2496         int from, to, merge, todo;
2497         struct skb_frag_struct *fragfrom, *fragto;
2498 
2499         BUG_ON(shiftlen > skb->len);
2500         BUG_ON(skb_headlen(skb));       /* Would corrupt stream */
2501 
2502         todo = shiftlen;
2503         from = 0;
2504         to = skb_shinfo(tgt)->nr_frags;
2505         fragfrom = &skb_shinfo(skb)->frags[from];
2506 
2507         /* Actual merge is delayed until the point when we know we can
2508          * commit all, so that we don't have to undo partial changes
2509          */
2510         if (!to ||
2511             !skb_can_coalesce(tgt, to, skb_frag_page(fragfrom),
2512                               fragfrom->page_offset)) {
2513                 merge = -1;
2514         } else {
2515                 merge = to - 1;
2516 
2517                 todo -= skb_frag_size(fragfrom);
2518                 if (todo < 0) {
2519                         if (skb_prepare_for_shift(skb) ||
2520                             skb_prepare_for_shift(tgt))
2521                                 return 0;
2522 
2523                         /* All previous frag pointers might be stale! */
2524                         fragfrom = &skb_shinfo(skb)->frags[from];
2525                         fragto = &skb_shinfo(tgt)->frags[merge];
2526 
2527                         skb_frag_size_add(fragto, shiftlen);
2528                         skb_frag_size_sub(fragfrom, shiftlen);
2529                         fragfrom->page_offset += shiftlen;
2530 
2531                         goto onlymerged;
2532                 }
2533 
2534                 from++;
2535         }
2536 
2537         /* Skip full, not-fitting skb to avoid expensive operations */
2538         if ((shiftlen == skb->len) &&
2539             (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to))
2540                 return 0;
2541 
2542         if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt))
2543                 return 0;
2544 
2545         while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) {
2546                 if (to == MAX_SKB_FRAGS)
2547                         return 0;
2548 
2549                 fragfrom = &skb_shinfo(skb)->frags[from];
2550                 fragto = &skb_shinfo(tgt)->frags[to];
2551 
2552                 if (todo >= skb_frag_size(fragfrom)) {
2553                         *fragto = *fragfrom;
2554                         todo -= skb_frag_size(fragfrom);
2555                         from++;
2556                         to++;
2557 
2558                 } else {
2559                         __skb_frag_ref(fragfrom);
2560                         fragto->page = fragfrom->page;
2561                         fragto->page_offset = fragfrom->page_offset;
2562                         skb_frag_size_set(fragto, todo);
2563 
2564                         fragfrom->page_offset += todo;
2565                         skb_frag_size_sub(fragfrom, todo);
2566                         todo = 0;
2567 
2568                         to++;
2569                         break;
2570                 }
2571         }
2572 
2573         /* Ready to "commit" this state change to tgt */
2574         skb_shinfo(tgt)->nr_frags = to;
2575 
2576         if (merge >= 0) {
2577                 fragfrom = &skb_shinfo(skb)->frags[0];
2578                 fragto = &skb_shinfo(tgt)->frags[merge];
2579 
2580                 skb_frag_size_add(fragto, skb_frag_size(fragfrom));
2581                 __skb_frag_unref(fragfrom);
2582         }
2583 
2584         /* Reposition in the original skb */
2585         to = 0;
2586         while (from < skb_shinfo(skb)->nr_frags)
2587                 skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++];
2588         skb_shinfo(skb)->nr_frags = to;
2589 
2590         BUG_ON(todo > 0 && !skb_shinfo(skb)->nr_frags);
2591 
2592 onlymerged:
2593         /* Most likely the tgt won't ever need its checksum anymore, skb on
2594          * the other hand might need it if it needs to be resent
2595          */
2596         tgt->ip_summed = CHECKSUM_PARTIAL;
2597         skb->ip_summed = CHECKSUM_PARTIAL;
2598 
2599         /* Yak, is it really working this way? Some helper please? */
2600         skb->len -= shiftlen;
2601         skb->data_len -= shiftlen;
2602         skb->truesize -= shiftlen;
2603         tgt->len += shiftlen;
2604         tgt->data_len += shiftlen;
2605         tgt->truesize += shiftlen;
2606 
2607         return shiftlen;
2608 }
2609 
2610 /**
2611  * skb_prepare_seq_read - Prepare a sequential read of skb data
2612  * @skb: the buffer to read
2613  * @from: lower offset of data to be read
2614  * @to: upper offset of data to be read
2615  * @st: state variable
2616  *
2617  * Initializes the specified state variable. Must be called before
2618  * invoking skb_seq_read() for the first time.
2619  */
2620 void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
2621                           unsigned int to, struct skb_seq_state *st)
2622 {
2623         st->lower_offset = from;
2624         st->upper_offset = to;
2625         st->root_skb = st->cur_skb = skb;
2626         st->frag_idx = st->stepped_offset = 0;
2627         st->frag_data = NULL;
2628 }
2629 EXPORT_SYMBOL(skb_prepare_seq_read);
2630 
2631 /**
2632  * skb_seq_read - Sequentially read skb data
2633  * @consumed: number of bytes consumed by the caller so far
2634  * @data: destination pointer for data to be returned
2635  * @st: state variable
2636  *
2637  * Reads a block of skb data at @consumed relative to the
2638  * lower offset specified to skb_prepare_seq_read(). Assigns
2639  * the head of the data block to @data and returns the length
2640  * of the block or 0 if the end of the skb data or the upper
2641  * offset has been reached.
2642  *
2643  * The caller is not required to consume all of the data
2644  * returned, i.e. @consumed is typically set to the number
2645  * of bytes already consumed and the next call to
2646  * skb_seq_read() will return the remaining part of the block.
2647  *
2648  * Note 1: The size of each block of data returned can be arbitrary,
2649  *       this limitation is the cost for zerocopy seqeuental
2650  *       reads of potentially non linear data.
2651  *
2652  * Note 2: Fragment lists within fragments are not implemented
2653  *       at the moment, state->root_skb could be replaced with
2654  *       a stack for this purpose.
2655  */
2656 unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
2657                           struct skb_seq_state *st)
2658 {
2659         unsigned int block_limit, abs_offset = consumed + st->lower_offset;
2660         skb_frag_t *frag;
2661 
2662         if (unlikely(abs_offset >= st->upper_offset)) {
2663                 if (st->frag_data) {
2664                         kunmap_atomic(st->frag_data);
2665                         st->frag_data = NULL;
2666                 }
2667                 return 0;
2668         }
2669 
2670 next_skb:
2671         block_limit = skb_headlen(st->cur_skb) + st->stepped_offset;
2672 
2673         if (abs_offset < block_limit && !st->frag_data) {
2674                 *data = st->cur_skb->data + (abs_offset - st->stepped_offset);
2675                 return block_limit - abs_offset;
2676         }
2677 
2678         if (st->frag_idx == 0 && !st->frag_data)
2679                 st->stepped_offset += skb_headlen(st->cur_skb);
2680 
2681         while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) {
2682                 frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx];
2683                 block_limit = skb_frag_size(frag) + st->stepped_offset;
2684 
2685                 if (abs_offset < block_limit) {
2686                         if (!st->frag_data)
2687                                 st->frag_data = kmap_atomic(skb_frag_page(frag));
2688 
2689                         *data = (u8 *) st->frag_data + frag->page_offset +
2690                                 (abs_offset - st->stepped_offset);
2691 
2692                         return block_limit - abs_offset;
2693                 }
2694 
2695                 if (st->frag_data) {
2696                         kunmap_atomic(st->frag_data);
2697                         st->frag_data = NULL;
2698                 }
2699 
2700                 st->frag_idx++;
2701                 st->stepped_offset += skb_frag_size(frag);
2702         }
2703 
2704         if (st->frag_data) {
2705                 kunmap_atomic(st->frag_data);
2706                 st->frag_data = NULL;
2707         }
2708 
2709         if (st->root_skb == st->cur_skb && skb_has_frag_list(st->root_skb)) {
2710                 st->cur_skb = skb_shinfo(st->root_skb)->frag_list;
2711                 st->frag_idx = 0;
2712                 goto next_skb;
2713         } else if (st->cur_skb->next) {
2714                 st->cur_skb = st->cur_skb->next;
2715                 st->frag_idx = 0;
2716                 goto next_skb;
2717         }
2718 
2719         return 0;
2720 }
2721 EXPORT_SYMBOL(skb_seq_read);
2722 
2723 /**
2724  * skb_abort_seq_read - Abort a sequential read of skb data
2725  * @st: state variable
2726  *
2727  * Must be called if skb_seq_read() was not called until it
2728  * returned 0.
2729  */
2730 void skb_abort_seq_read(struct skb_seq_state *st)
2731 {
2732         if (st->frag_data)
2733                 kunmap_atomic(st->frag_data);
2734 }
2735 EXPORT_SYMBOL(skb_abort_seq_read);
2736 
2737 #define TS_SKB_CB(state)        ((struct skb_seq_state *) &((state)->cb))
2738 
2739 static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text,
2740                                           struct ts_config *conf,
2741                                           struct ts_state *state)
2742 {
2743         return skb_seq_read(offset, text, TS_SKB_CB(state));
2744 }
2745 
2746 static void skb_ts_finish(struct ts_config *conf, struct ts_state *state)
2747 {
2748         skb_abort_seq_read(TS_SKB_CB(state));
2749 }
2750 
2751 /**
2752  * skb_find_text - Find a text pattern in skb data
2753  * @skb: the buffer to look in
2754  * @from: search offset
2755  * @to: search limit
2756  * @config: textsearch configuration
2757  * @state: uninitialized textsearch state variable
2758  *
2759  * Finds a pattern in the skb data according to the specified
2760  * textsearch configuration. Use textsearch_next() to retrieve
2761  * subsequent occurrences of the pattern. Returns the offset
2762  * to the first occurrence or UINT_MAX if no match was found.
2763  */
2764 unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
2765                            unsigned int to, struct ts_config *config,
2766                            struct ts_state *state)
2767 {
2768         unsigned int ret;
2769 
2770         config->get_next_block = skb_ts_get_next_block;
2771         config->finish = skb_ts_finish;
2772 
2773         skb_prepare_seq_read(skb, from, to, TS_SKB_CB(state));
2774 
2775         ret = textsearch_find(config, state);
2776         return (ret <= to - from ? ret : UINT_MAX);
2777 }
2778 EXPORT_SYMBOL(skb_find_text);
2779 
2780 /**
2781  * skb_append_datato_frags - append the user data to a skb
2782  * @sk: sock  structure
2783  * @skb: skb structure to be appened with user data.
2784  * @getfrag: call back function to be used for getting the user data
2785  * @from: pointer to user message iov
2786  * @length: length of the iov message
2787  *
2788  * Description: This procedure append the user data in the fragment part
2789  * of the skb if any page alloc fails user this procedure returns  -ENOMEM
2790  */
2791 int skb_append_datato_frags(struct sock *sk, struct sk_buff *skb,
2792                         int (*getfrag)(void *from, char *to, int offset,
2793                                         int len, int odd, struct sk_buff *skb),
2794                         void *from, int length)
2795 {
2796         int frg_cnt = skb_shinfo(skb)->nr_frags;
2797         int copy;
2798         int offset = 0;
2799         int ret;
2800         struct page_frag *pfrag = &current->task_frag;
2801 
2802         do {
2803                 /* Return error if we don't have space for new frag */
2804                 if (frg_cnt >= MAX_SKB_FRAGS)
2805                         return -EMSGSIZE;
2806 
2807                 if (!sk_page_frag_refill(sk, pfrag))
2808                         return -ENOMEM;
2809 
2810                 /* copy the user data to page */
2811                 copy = min_t(int, length, pfrag->size - pfrag->offset);
2812 
2813                 ret = getfrag(from, page_address(pfrag->page) + pfrag->offset,
2814                               offset, copy, 0, skb);
2815                 if (ret < 0)
2816                         return -EFAULT;
2817 
2818                 /* copy was successful so update the size parameters */
2819                 skb_fill_page_desc(skb, frg_cnt, pfrag->page, pfrag->offset,
2820                                    copy);
2821                 frg_cnt++;
2822                 pfrag->offset += copy;
2823                 get_page(pfrag->page);
2824 
2825                 skb->truesize += copy;
2826                 atomic_add(copy, &sk->sk_wmem_alloc);
2827                 skb->len += copy;
2828                 skb->data_len += copy;
2829                 offset += copy;
2830                 length -= copy;
2831 
2832         } while (length > 0);
2833 
2834         return 0;
2835 }
2836 EXPORT_SYMBOL(skb_append_datato_frags);
2837 
2838 /**
2839  *      skb_pull_rcsum - pull skb and update receive checksum
2840  *      @skb: buffer to update
2841  *      @len: length of data pulled
2842  *
2843  *      This function performs an skb_pull on the packet and updates
2844  *      the CHECKSUM_COMPLETE checksum.  It should be used on
2845  *      receive path processing instead of skb_pull unless you know
2846  *      that the checksum difference is zero (e.g., a valid IP header)
2847  *      or you are setting ip_summed to CHECKSUM_NONE.
2848  */
2849 unsigned char *skb_pull_rcsum(struct sk_buff *skb, unsigned int len)
2850 {
2851         BUG_ON(len > skb->len);
2852         skb->len -= len;
2853         BUG_ON(skb->len < skb->data_len);
2854         skb_postpull_rcsum(skb, skb->data, len);
2855         return skb->data += len;
2856 }
2857 EXPORT_SYMBOL_GPL(skb_pull_rcsum);
2858 
2859 /**
2860  *      skb_segment - Perform protocol segmentation on skb.
2861  *      @head_skb: buffer to segment
2862  *      @features: features for the output path (see dev->features)
2863  *
2864  *      This function performs segmentation on the given skb.  It returns
2865  *      a pointer to the first in a list of new skbs for the segments.
2866  *      In case of error it returns ERR_PTR(err).
2867  */
2868 struct sk_buff *skb_segment(struct sk_buff *head_skb,
2869                             netdev_features_t features)
2870 {
2871         struct sk_buff *segs = NULL;
2872         struct sk_buff *tail = NULL;
2873         struct sk_buff *list_skb = skb_shinfo(head_skb)->frag_list;
2874         skb_frag_t *frag = skb_shinfo(head_skb)->frags;
2875         unsigned int mss = skb_shinfo(head_skb)->gso_size;
2876         unsigned int doffset = head_skb->data - skb_mac_header(head_skb);
2877         struct sk_buff *frag_skb = head_skb;
2878         unsigned int offset = doffset;
2879         unsigned int tnl_hlen = skb_tnl_header_len(head_skb);
2880         unsigned int headroom;
2881         unsigned int len;
2882         __be16 proto;
2883         bool csum;
2884         int sg = !!(features & NETIF_F_SG);
2885         int nfrags = skb_shinfo(head_skb)->nr_frags;
2886         int err = -ENOMEM;
2887         int i = 0;
2888         int pos;
2889         int dummy;
2890 
2891         __skb_push(head_skb, doffset);
2892         proto = skb_network_protocol(head_skb, &dummy);
2893         if (unlikely(!proto))
2894                 return ERR_PTR(-EINVAL);
2895 
2896         csum = !head_skb->encap_hdr_csum &&
2897             !!can_checksum_protocol(features, proto);
2898 
2899         headroom = skb_headroom(head_skb);
2900         pos = skb_headlen(head_skb);
2901 
2902         do {
2903                 struct sk_buff *nskb;
2904                 skb_frag_t *nskb_frag;
2905                 int hsize;
2906                 int size;
2907 
2908                 len = head_skb->len - offset;
2909                 if (len > mss)
2910                         len = mss;
2911 
2912                 hsize = skb_headlen(head_skb) - offset;
2913                 if (hsize < 0)
2914                         hsize = 0;
2915                 if (hsize > len || !sg)
2916                         hsize = len;
2917 
2918                 if (!hsize && i >= nfrags && skb_headlen(list_skb) &&
2919                     (skb_headlen(list_skb) == len || sg)) {
2920                         BUG_ON(skb_headlen(list_skb) > len);
2921 
2922                         i = 0;
2923                         nfrags = skb_shinfo(list_skb)->nr_frags;
2924                         frag = skb_shinfo(list_skb)->frags;
2925                         frag_skb = list_skb;
2926                         pos += skb_headlen(list_skb);
2927 
2928                         while (pos < offset + len) {
2929                                 BUG_ON(i >= nfrags);
2930 
2931                                 size = skb_frag_size(frag);
2932                                 if (pos + size > offset + len)
2933                                         break;
2934 
2935                                 i++;
2936                                 pos += size;
2937                                 frag++;
2938                         }
2939 
2940                         nskb = skb_clone(list_skb, GFP_ATOMIC);
2941                         list_skb = list_skb->next;
2942 
2943                         if (unlikely(!nskb))
2944                                 goto err;
2945 
2946                         if (unlikely(pskb_trim(nskb, len))) {
2947                                 kfree_skb(nskb);
2948                                 goto err;
2949                         }
2950 
2951                         hsize = skb_end_offset(nskb);
2952                         if (skb_cow_head(nskb, doffset + headroom)) {
2953                                 kfree_skb(nskb);
2954                                 goto err;
2955                         }
2956 
2957                         nskb->truesize += skb_end_offset(nskb) - hsize;
2958                         skb_release_head_state(nskb);
2959                         __skb_push(nskb, doffset);
2960                 } else {
2961                         nskb = __alloc_skb(hsize + doffset + headroom,
2962                                            GFP_ATOMIC, skb_alloc_rx_flag(head_skb),
2963                                            NUMA_NO_NODE);
2964 
2965                         if (unlikely(!nskb))
2966                                 goto err;
2967 
2968                         skb_reserve(nskb, headroom);
2969                         __skb_put(nskb, doffset);
2970                 }
2971 
2972                 if (segs)
2973                         tail->next = nskb;
2974                 else
2975                         segs = nskb;
2976                 tail = nskb;
2977 
2978                 __copy_skb_header(nskb, head_skb);
2979                 nskb->mac_len = head_skb->mac_len;
2980 
2981                 skb_headers_offset_update(nskb, skb_headroom(nskb) - headroom);
2982 
2983                 skb_copy_from_linear_data_offset(head_skb, -tnl_hlen,
2984                                                  nskb->data - tnl_hlen,
2985                                                  doffset + tnl_hlen);
2986 
2987                 if (nskb->len == len + doffset)
2988                         goto perform_csum_check;
2989 
2990                 if (!sg) {
2991                         nskb->ip_summed = CHECKSUM_NONE;
2992                         nskb->csum = skb_copy_and_csum_bits(head_skb, offset,
2993                                                             skb_put(nskb, len),
2994                                                             len, 0);
2995                         SKB_GSO_CB(nskb)->csum_start =
2996                             skb_headroom(nskb) + doffset;
2997                         continue;
2998                 }
2999 
3000                 nskb_frag = skb_shinfo(nskb)->frags;
3001 
3002                 skb_copy_from_linear_data_offset(head_skb, offset,
3003                                                  skb_put(nskb, hsize), hsize);
3004 
3005                 skb_shinfo(nskb)->tx_flags = skb_shinfo(head_skb)->tx_flags &
3006                         SKBTX_SHARED_FRAG;
3007 
3008                 while (pos < offset + len) {
3009                         if (i >= nfrags) {
3010                                 BUG_ON(skb_headlen(list_skb));
3011 
3012                                 i = 0;
3013                                 nfrags = skb_shinfo(list_skb)->nr_frags;
3014                                 frag = skb_shinfo(list_skb)->frags;
3015                                 frag_skb = list_skb;
3016 
3017                                 BUG_ON(!nfrags);
3018 
3019                                 list_skb = list_skb->next;
3020                         }
3021 
3022                         if (unlikely(skb_shinfo(nskb)->nr_frags >=
3023                                      MAX_SKB_FRAGS)) {
3024                                 net_warn_ratelimited(
3025                                         "skb_segment: too many frags: %u %u\n",
3026                                         pos, mss);
3027                                 goto err;
3028                         }
3029 
3030                         if (unlikely(skb_orphan_frags(frag_skb, GFP_ATOMIC)))
3031                                 goto err;
3032 
3033                         *nskb_frag = *frag;
3034                         __skb_frag_ref(nskb_frag);
3035                         size = skb_frag_size(nskb_frag);
3036 
3037                         if (pos < offset) {
3038                                 nskb_frag->page_offset += offset - pos;
3039                                 skb_frag_size_sub(nskb_frag, offset - pos);
3040                         }
3041 
3042                         skb_shinfo(nskb)->nr_frags++;
3043 
3044                         if (pos + size <= offset + len) {
3045                                 i++;
3046                                 frag++;
3047                                 pos += size;
3048                         } else {
3049                                 skb_frag_size_sub(nskb_frag, pos + size - (offset + len));
3050                                 goto skip_fraglist;
3051                         }
3052 
3053                         nskb_frag++;
3054                 }
3055 
3056 skip_fraglist:
3057                 nskb->data_len = len - hsize;
3058                 nskb->len += nskb->data_len;
3059                 nskb->truesize += nskb->data_len;
3060 
3061 perform_csum_check:
3062                 if (!csum) {
3063                         nskb->csum = skb_checksum(nskb, doffset,
3064                                                   nskb->len - doffset, 0);
3065                         nskb->ip_summed = CHECKSUM_NONE;
3066                         SKB_GSO_CB(nskb)->csum_start =
3067                             skb_headroom(nskb) + doffset;
3068                 }
3069         } while ((offset += len) < head_skb->len);
3070 
3071         return segs;
3072 
3073 err:
3074         kfree_skb_list(segs);
3075         return ERR_PTR(err);
3076 }
3077 EXPORT_SYMBOL_GPL(skb_segment);
3078 
3079 int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb)
3080 {
3081         struct skb_shared_info *pinfo, *skbinfo = skb_shinfo(skb);
3082         unsigned int offset = skb_gro_offset(skb);
3083         unsigned int headlen = skb_headlen(skb);
3084         struct sk_buff *nskb, *lp, *p = *head;
3085         unsigned int len = skb_gro_len(skb);
3086         unsigned int delta_truesize;
3087         unsigned int headroom;
3088 
3089         if (unlikely(p->len + len >= 65536))
3090                 return -E2BIG;
3091 
3092         lp = NAPI_GRO_CB(p)->last;
3093         pinfo = skb_shinfo(lp);
3094 
3095         if (headlen <= offset) {
3096                 skb_frag_t *frag;
3097                 skb_frag_t *frag2;
3098                 int i = skbinfo->nr_frags;
3099                 int nr_frags = pinfo->nr_frags + i;
3100 
3101                 if (nr_frags > MAX_SKB_FRAGS)
3102                         goto merge;
3103 
3104                 offset -= headlen;
3105                 pinfo->nr_frags = nr_frags;
3106                 skbinfo->nr_frags = 0;
3107 
3108                 frag = pinfo->frags + nr_frags;
3109                 frag2 = skbinfo->frags + i;
3110                 do {
3111                         *--frag = *--frag2;
3112                 } while (--i);
3113 
3114                 frag->page_offset += offset;
3115                 skb_frag_size_sub(frag, offset);
3116 
3117                 /* all fragments truesize : remove (head size + sk_buff) */
3118                 delta_truesize = skb->truesize -
3119                                  SKB_TRUESIZE(skb_end_offset(skb));
3120 
3121                 skb->truesize -= skb->data_len;
3122                 skb->len -= skb->data_len;
3123                 skb->data_len = 0;
3124 
3125                 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE;
3126                 goto done;
3127         } else if (skb->head_frag) {
3128                 int nr_frags = pinfo->nr_frags;
3129                 skb_frag_t *frag = pinfo->frags + nr_frags;
3130                 struct page *page = virt_to_head_page(skb->head);
3131                 unsigned int first_size = headlen - offset;
3132                 unsigned int first_offset;
3133 
3134                 if (nr_frags + 1 + skbinfo->nr_frags > MAX_SKB_FRAGS)
3135                         goto merge;
3136 
3137                 first_offset = skb->data -
3138                                (unsigned char *)page_address(page) +
3139                                offset;
3140 
3141                 pinfo->nr_frags = nr_frags + 1 + skbinfo->nr_frags;
3142 
3143                 frag->page.p      = page;
3144                 frag->page_offset = first_offset;
3145                 skb_frag_size_set(frag, first_size);
3146 
3147                 memcpy(frag + 1, skbinfo->frags, sizeof(*frag) * skbinfo->nr_frags);
3148                 /* We dont need to clear skbinfo->nr_frags here */
3149 
3150                 delta_truesize = skb->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff));
3151                 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE_STOLEN_HEAD;
3152                 goto done;
3153         }
3154         if (pinfo->frag_list)
3155                 goto merge;
3156         if (skb_gro_len(p) != pinfo->gso_size)
3157                 return -E2BIG;
3158 
3159         headroom = skb_headroom(p);
3160         nskb = alloc_skb(headroom + skb_gro_offset(p), GFP_ATOMIC);
3161         if (unlikely(!nskb))
3162                 return -ENOMEM;
3163 
3164         __copy_skb_header(nskb, p);
3165         nskb->mac_len = p->mac_len;
3166 
3167         skb_reserve(nskb, headroom);
3168         __skb_put(nskb, skb_gro_offset(p));
3169 
3170         skb_set_mac_header(nskb, skb_mac_header(p) - p->data);
3171         skb_set_network_header(nskb, skb_network_offset(p));
3172         skb_set_transport_header(nskb, skb_transport_offset(p));
3173 
3174         __skb_pull(p, skb_gro_offset(p));
3175         memcpy(skb_mac_header(nskb), skb_mac_header(p),
3176                p->data - skb_mac_header(p));
3177 
3178         skb_shinfo(nskb)->frag_list = p;
3179         skb_shinfo(nskb)->gso_size = pinfo->gso_size;
3180         pinfo->gso_size = 0;
3181         skb_header_release(p);
3182         NAPI_GRO_CB(nskb)->last = p;
3183 
3184         nskb->data_len += p->len;
3185         nskb->truesize += p->truesize;
3186         nskb->len += p->len;
3187 
3188         *head = nskb;
3189         nskb->next = p->next;
3190         p->next = NULL;
3191 
3192         p = nskb;
3193 
3194 merge:
3195         delta_truesize = skb->truesize;
3196         if (offset > headlen) {
3197                 unsigned int eat = offset - headlen;
3198 
3199                 skbinfo->frags[0].page_offset += eat;
3200                 skb_frag_size_sub(&skbinfo->frags[0], eat);
3201                 skb->data_len -= eat;
3202                 skb->len -= eat;
3203                 offset = headlen;
3204         }
3205 
3206         __skb_pull(skb, offset);
3207 
3208         if (NAPI_GRO_CB(p)->last == p)
3209                 skb_shinfo(p)->frag_list = skb;
3210         else
3211                 NAPI_GRO_CB(p)->last->next = skb;
3212         NAPI_GRO_CB(p)->last = skb;
3213         skb_header_release(skb);
3214         lp = p;
3215 
3216 done:
3217         NAPI_GRO_CB(p)->count++;
3218         p->data_len += len;
3219         p->truesize += delta_truesize;
3220         p->len += len;
3221         if (lp != p) {
3222                 lp->data_len += len;
3223                 lp->truesize += delta_truesize;
3224                 lp->len += len;
3225         }
3226         NAPI_GRO_CB(skb)->same_flow = 1;
3227         return 0;
3228 }
3229 EXPORT_SYMBOL_GPL(skb_gro_receive);
3230 
3231 void __init skb_init(void)
3232 {
3233         skbuff_head_cache = kmem_cache_create("skbuff_head_cache",
3234                                               sizeof(struct sk_buff),
3235                                               0,
3236                                               SLAB_HWCACHE_ALIGN|SLAB_PANIC,
3237                                               NULL);
3238         skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache",
3239                                                 (2*sizeof(struct sk_buff)) +
3240                                                 sizeof(atomic_t),
3241                                                 0,
3242                                                 SLAB_HWCACHE_ALIGN|SLAB_PANIC,
3243                                                 NULL);
3244 }
3245 
3246 /**
3247  *      skb_to_sgvec - Fill a scatter-gather list from a socket buffer
3248  *      @skb: Socket buffer containing the buffers to be mapped
3249  *      @sg: The scatter-gather list to map into
3250  *      @offset: The offset into the buffer's contents to start mapping
3251  *      @len: Length of buffer space to be mapped
3252  *
3253  *      Fill the specified scatter-gather list with mappings/pointers into a
3254  *      region of the buffer space attached to a socket buffer.
3255  */
3256 static int
3257 __skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len)
3258 {
3259         int start = skb_headlen(skb);
3260         int i, copy = start - offset;
3261         struct sk_buff *frag_iter;
3262         int elt = 0;
3263 
3264         if (copy > 0) {
3265                 if (copy > len)
3266                         copy = len;
3267                 sg_set_buf(sg, skb->data + offset, copy);
3268                 elt++;
3269                 if ((len -= copy) == 0)
3270                         return elt;
3271                 offset += copy;
3272         }
3273 
3274         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3275                 int end;
3276 
3277                 WARN_ON(start > offset + len);
3278 
3279                 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
3280                 if ((copy = end - offset) > 0) {
3281                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3282 
3283                         if (copy > len)
3284                                 copy = len;
3285                         sg_set_page(&sg[elt], skb_frag_page(frag), copy,
3286                                         frag->page_offset+offset-start);
3287                         elt++;
3288                         if (!(len -= copy))
3289                                 return elt;
3290                         offset += copy;
3291                 }
3292                 start = end;
3293         }
3294 
3295         skb_walk_frags(skb, frag_iter) {
3296                 int end;
3297 
3298                 WARN_ON(start > offset + len);
3299 
3300                 end = start + frag_iter->len;
3301                 if ((copy = end - offset) > 0) {
3302                         if (copy > len)
3303                                 copy = len;
3304                         elt += __skb_to_sgvec(frag_iter, sg+elt, offset - start,
3305                                               copy);
3306                         if ((len -= copy) == 0)
3307                                 return elt;
3308                         offset += copy;
3309                 }
3310                 start = end;
3311         }
3312         BUG_ON(len);
3313         return elt;
3314 }
3315 
3316 /* As compared with skb_to_sgvec, skb_to_sgvec_nomark only map skb to given
3317  * sglist without mark the sg which contain last skb data as the end.
3318  * So the caller can mannipulate sg list as will when padding new data after
3319  * the first call without calling sg_unmark_end to expend sg list.
3320  *
3321  * Scenario to use skb_to_sgvec_nomark:
3322  * 1. sg_init_table
3323  * 2. skb_to_sgvec_nomark(payload1)
3324  * 3. skb_to_sgvec_nomark(payload2)
3325  *
3326  * This is equivalent to:
3327  * 1. sg_init_table
3328  * 2. skb_to_sgvec(payload1)
3329  * 3. sg_unmark_end
3330  * 4. skb_to_sgvec(payload2)
3331  *
3332  * When mapping mutilple payload conditionally, skb_to_sgvec_nomark
3333  * is more preferable.
3334  */
3335 int skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg,
3336                         int offset, int len)
3337 {
3338         return __skb_to_sgvec(skb, sg, offset, len);
3339 }
3340 EXPORT_SYMBOL_GPL(skb_to_sgvec_nomark);
3341 
3342 int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len)
3343 {
3344         int nsg = __skb_to_sgvec(skb, sg, offset, len);
3345 
3346         sg_mark_end(&sg[nsg - 1]);
3347 
3348         return nsg;
3349 }
3350 EXPORT_SYMBOL_GPL(skb_to_sgvec);
3351 
3352 /**
3353  *      skb_cow_data - Check that a socket buffer's data buffers are writable
3354  *      @skb: The socket buffer to check.
3355  *      @tailbits: Amount of trailing space to be added
3356  *      @trailer: Returned pointer to the skb where the @tailbits space begins
3357  *
3358  *      Make sure that the data buffers attached to a socket buffer are
3359  *      writable. If they are not, private copies are made of the data buffers
3360  *      and the socket buffer is set to use these instead.
3361  *
3362  *      If @tailbits is given, make sure that there is space to write @tailbits
3363  *      bytes of data beyond current end of socket buffer.  @trailer will be
3364  *      set to point to the skb in which this space begins.
3365  *
3366  *      The number of scatterlist elements required to completely map the
3367  *      COW'd and extended socket buffer will be returned.
3368  */
3369 int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer)
3370 {
3371         int copyflag;
3372         int elt;
3373         struct sk_buff *skb1, **skb_p;
3374 
3375         /* If skb is cloned or its head is paged, reallocate
3376          * head pulling out all the pages (pages are considered not writable
3377          * at the moment even if they are anonymous).
3378          */
3379         if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) &&
3380             __pskb_pull_tail(skb, skb_pagelen(skb)-skb_headlen(skb)) == NULL)
3381                 return -ENOMEM;
3382 
3383         /* Easy case. Most of packets will go this way. */
3384         if (!skb_has_frag_list(skb)) {
3385                 /* A little of trouble, not enough of space for trailer.
3386                  * This should not happen, when stack is tuned to generate
3387                  * good frames. OK, on miss we reallocate and reserve even more
3388                  * space, 128 bytes is fair. */
3389 
3390                 if (skb_tailroom(skb) < tailbits &&
3391                     pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC))
3392                         return -ENOMEM;
3393 
3394                 /* Voila! */
3395                 *trailer = skb;
3396                 return 1;
3397         }
3398 
3399         /* Misery. We are in troubles, going to mincer fragments... */
3400 
3401         elt = 1;
3402         skb_p = &skb_shinfo(skb)->frag_list;
3403         copyflag = 0;
3404 
3405         while ((skb1 = *skb_p) != NULL) {
3406                 int ntail = 0;
3407 
3408                 /* The fragment is partially pulled by someone,
3409                  * this can happen on input. Copy it and everything
3410                  * after it. */
3411 
3412                 if (skb_shared(skb1))
3413                         copyflag = 1;
3414 
3415                 /* If the skb is the last, worry about trailer. */
3416 
3417                 if (skb1->next == NULL && tailbits) {
3418                         if (skb_shinfo(skb1)->nr_frags ||
3419                             skb_has_frag_list(skb1) ||
3420                             skb_tailroom(skb1) < tailbits)
3421                                 ntail = tailbits + 128;
3422                 }
3423 
3424                 if (copyflag ||
3425                     skb_cloned(skb1) ||
3426                     ntail ||
3427                     skb_shinfo(skb1)->nr_frags ||
3428                     skb_has_frag_list(skb1)) {
3429                         struct sk_buff *skb2;
3430 
3431                         /* Fuck, we are miserable poor guys... */
3432                         if (ntail == 0)
3433                                 skb2 = skb_copy(skb1, GFP_ATOMIC);
3434                         else
3435                                 skb2 = skb_copy_expand(skb1,
3436                                                        skb_headroom(skb1),
3437                                                        ntail,
3438                                                        GFP_ATOMIC);
3439                         if (unlikely(skb2 == NULL))
3440                                 return -ENOMEM;
3441 
3442                         if (skb1->sk)
3443                                 skb_set_owner_w(skb2, skb1->sk);
3444 
3445                         /* Looking around. Are we still alive?
3446                          * OK, link new skb, drop old one */
3447 
3448                         skb2->next = skb1->next;
3449                         *skb_p = skb2;
3450                         kfree_skb(skb1);
3451                         skb1 = skb2;
3452                 }
3453                 elt++;
3454                 *trailer = skb1;
3455                 skb_p = &skb1->next;
3456         }
3457 
3458         return elt;
3459 }
3460 EXPORT_SYMBOL_GPL(skb_cow_data);
3461 
3462 static void sock_rmem_free(struct sk_buff *skb)
3463 {
3464         struct sock *sk = skb->sk;
3465 
3466         atomic_sub(skb->truesize, &sk->sk_rmem_alloc);
3467 }
3468 
3469 /*
3470  * Note: We dont mem charge error packets (no sk_forward_alloc changes)
3471  */
3472 int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb)
3473 {
3474         if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >=
3475             (unsigned int)sk->sk_rcvbuf)
3476                 return -ENOMEM;
3477 
3478         skb_orphan(skb);
3479         skb->sk = sk;
3480         skb->destructor = sock_rmem_free;
3481         atomic_add(skb->truesize, &sk->sk_rmem_alloc);
3482 
3483         /* before exiting rcu section, make sure dst is refcounted */
3484         skb_dst_force(skb);
3485 
3486         skb_queue_tail(&sk->sk_error_queue, skb);
3487         if (!sock_flag(sk, SOCK_DEAD))
3488                 sk->sk_data_ready(sk);
3489         return 0;
3490 }
3491 EXPORT_SYMBOL(sock_queue_err_skb);
3492 
3493 void skb_tstamp_tx(struct sk_buff *orig_skb,
3494                 struct skb_shared_hwtstamps *hwtstamps)
3495 {
3496         struct sock *sk = orig_skb->sk;
3497         struct sock_exterr_skb *serr;
3498         struct sk_buff *skb;
3499         int err;
3500 
3501         if (!sk)
3502                 return;
3503 
3504         if (hwtstamps) {
3505                 *skb_hwtstamps(orig_skb) =
3506                         *hwtstamps;
3507         } else {
3508                 /*
3509                  * no hardware time stamps available,
3510                  * so keep the shared tx_flags and only
3511                  * store software time stamp
3512                  */
3513                 orig_skb->tstamp = ktime_get_real();
3514         }
3515 
3516         skb = skb_clone(orig_skb, GFP_ATOMIC);
3517         if (!skb)
3518                 return;
3519 
3520         serr = SKB_EXT_ERR(skb);
3521         memset(serr, 0, sizeof(*serr));
3522         serr->ee.ee_errno = ENOMSG;
3523         serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING;
3524 
3525         err = sock_queue_err_skb(sk, skb);
3526 
3527         if (err)
3528                 kfree_skb(skb);
3529 }
3530 EXPORT_SYMBOL_GPL(skb_tstamp_tx);
3531 
3532 void skb_complete_wifi_ack(struct sk_buff *skb, bool acked)
3533 {
3534         struct sock *sk = skb->sk;
3535         struct sock_exterr_skb *serr;
3536         int err;
3537 
3538         skb->wifi_acked_valid = 1;
3539         skb->wifi_acked = acked;
3540 
3541         serr = SKB_EXT_ERR(skb);
3542         memset(serr, 0, sizeof(*serr));
3543         serr->ee.ee_errno = ENOMSG;
3544         serr->ee.ee_origin = SO_EE_ORIGIN_TXSTATUS;
3545 
3546         err = sock_queue_err_skb(sk, skb);
3547         if (err)
3548                 kfree_skb(skb);
3549 }
3550 EXPORT_SYMBOL_GPL(skb_complete_wifi_ack);
3551 
3552 
3553 /**
3554  * skb_partial_csum_set - set up and verify partial csum values for packet
3555  * @skb: the skb to set
3556  * @start: the number of bytes after skb->data to start checksumming.
3557  * @off: the offset from start to place the checksum.
3558  *
3559  * For untrusted partially-checksummed packets, we need to make sure the values
3560  * for skb->csum_start and skb->csum_offset are valid so we don't oops.
3561  *
3562  * This function checks and sets those values and skb->ip_summed: if this
3563  * returns false you should drop the packet.
3564  */
3565 bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off)
3566 {
3567         if (unlikely(start > skb_headlen(skb)) ||
3568             unlikely((int)start + off > skb_headlen(skb) - 2)) {
3569                 net_warn_ratelimited("bad partial csum: csum=%u/%u len=%u\n",
3570                                      start, off, skb_headlen(skb));
3571                 return false;
3572         }
3573         skb->ip_summed = CHECKSUM_PARTIAL;
3574         skb->csum_start = skb_headroom(skb) + start;
3575         skb->csum_offset = off;
3576         skb_set_transport_header(skb, start);
3577         return true;
3578 }
3579 EXPORT_SYMBOL_GPL(skb_partial_csum_set);
3580 
3581 static int skb_maybe_pull_tail(struct sk_buff *skb, unsigned int len,
3582                                unsigned int max)
3583 {
3584         if (skb_headlen(skb) >= len)
3585                 return 0;
3586 
3587         /* If we need to pullup then pullup to the max, so we
3588          * won't need to do it again.
3589          */
3590         if (max > skb->len)
3591                 max = skb->len;
3592 
3593         if (__pskb_pull_tail(skb, max - skb_headlen(skb)) == NULL)
3594                 return -ENOMEM;
3595 
3596         if (skb_headlen(skb) < len)
3597                 return -EPROTO;
3598 
3599         return 0;
3600 }
3601 
3602 #define MAX_TCP_HDR_LEN (15 * 4)
3603 
3604 static __sum16 *skb_checksum_setup_ip(struct sk_buff *skb,
3605                                       typeof(IPPROTO_IP) proto,
3606                                       unsigned int off)
3607 {
3608         switch (proto) {
3609                 int err;
3610 
3611         case IPPROTO_TCP:
3612                 err = skb_maybe_pull_tail(skb, off + sizeof(struct tcphdr),
3613                                           off + MAX_TCP_HDR_LEN);
3614                 if (!err && !skb_partial_csum_set(skb, off,
3615                                                   offsetof(struct tcphdr,
3616                                                            check)))
3617                         err = -EPROTO;
3618                 return err ? ERR_PTR(err) : &tcp_hdr(skb)->check;
3619 
3620         case IPPROTO_UDP:
3621                 err = skb_maybe_pull_tail(skb, off + sizeof(struct udphdr),
3622                                           off + sizeof(struct udphdr));
3623                 if (!err && !skb_partial_csum_set(skb, off,
3624                                                   offsetof(struct udphdr,
3625                                                            check)))
3626                         err = -EPROTO;
3627                 return err ? ERR_PTR(err) : &udp_hdr(skb)->check;
3628         }
3629 
3630         return ERR_PTR(-EPROTO);
3631 }
3632 
3633 /* This value should be large enough to cover a tagged ethernet header plus
3634  * maximally sized IP and TCP or UDP headers.
3635  */
3636 #define MAX_IP_HDR_LEN 128
3637 
3638 static int skb_checksum_setup_ipv4(struct sk_buff *skb, bool recalculate)
3639 {
3640         unsigned int off;
3641         bool fragment;
3642         __sum16 *csum;
3643         int err;
3644 
3645         fragment = false;
3646 
3647         err = skb_maybe_pull_tail(skb,
3648                                   sizeof(struct iphdr),
3649                                   MAX_IP_HDR_LEN);
3650         if (err < 0)
3651                 goto out;
3652 
3653         if (ip_hdr(skb)->frag_off & htons(IP_OFFSET | IP_MF))
3654                 fragment = true;
3655 
3656         off = ip_hdrlen(skb);
3657 
3658         err = -EPROTO;
3659 
3660         if (fragment)
3661                 goto out;
3662 
3663         csum = skb_checksum_setup_ip(skb, ip_hdr(skb)->protocol, off);
3664         if (IS_ERR(csum))
3665                 return PTR_ERR(csum);
3666 
3667         if (recalculate)
3668                 *csum = ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
3669                                            ip_hdr(skb)->daddr,
3670                                            skb->len - off,
3671                                            ip_hdr(skb)->protocol, 0);
3672         err = 0;
3673 
3674 out:
3675         return err;
3676 }
3677 
3678 /* This value should be large enough to cover a tagged ethernet header plus
3679  * an IPv6 header, all options, and a maximal TCP or UDP header.
3680  */
3681 #define MAX_IPV6_HDR_LEN 256
3682 
3683 #define OPT_HDR(type, skb, off) \
3684         (type *)(skb_network_header(skb) + (off))
3685 
3686 static int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate)
3687 {
3688         int err;
3689         u8 nexthdr;
3690         unsigned int off;
3691         unsigned int len;
3692         bool fragment;
3693         bool done;
3694         __sum16 *csum;
3695 
3696         fragment = false;
3697         done = false;
3698 
3699         off = sizeof(struct ipv6hdr);
3700 
3701         err = skb_maybe_pull_tail(skb, off, MAX_IPV6_HDR_LEN);
3702         if (err < 0)
3703                 goto out;
3704 
3705         nexthdr = ipv6_hdr(skb)->nexthdr;
3706 
3707         len = sizeof(struct ipv6hdr) + ntohs(ipv6_hdr(skb)->payload_len);
3708         while (off <= len && !done) {
3709                 switch (nexthdr) {
3710                 case IPPROTO_DSTOPTS:
3711                 case IPPROTO_HOPOPTS:
3712                 case IPPROTO_ROUTING: {
3713                         struct ipv6_opt_hdr *hp;
3714 
3715                         err = skb_maybe_pull_tail(skb,
3716                                                   off +
3717                                                   sizeof(struct ipv6_opt_hdr),
3718                                                   MAX_IPV6_HDR_LEN);
3719                         if (err < 0)
3720                                 goto out;
3721 
3722                         hp = OPT_HDR(struct ipv6_opt_hdr, skb, off);
3723                         nexthdr = hp->nexthdr;
3724                         off += ipv6_optlen(hp);
3725                         break;
3726                 }
3727                 case IPPROTO_AH: {
3728                         struct ip_auth_hdr *hp;
3729 
3730                         err = skb_maybe_pull_tail(skb,
3731                                                   off +
3732                                                   sizeof(struct ip_auth_hdr),
3733                                                   MAX_IPV6_HDR_LEN);
3734                         if (err < 0)
3735                                 goto out;
3736 
3737                         hp = OPT_HDR(struct ip_auth_hdr, skb, off);
3738                         nexthdr = hp->nexthdr;
3739                         off += ipv6_authlen(hp);
3740                         break;
3741                 }
3742                 case IPPROTO_FRAGMENT: {
3743                         struct frag_hdr *hp;
3744 
3745                         err = skb_maybe_pull_tail(skb,
3746                                                   off +
3747                                                   sizeof(struct frag_hdr),
3748                                                   MAX_IPV6_HDR_LEN);
3749                         if (err < 0)
3750                                 goto out;
3751 
3752                         hp = OPT_HDR(struct frag_hdr, skb, off);
3753 
3754                         if (hp->frag_off & htons(IP6_OFFSET | IP6_MF))
3755                                 fragment = true;
3756 
3757                         nexthdr = hp->nexthdr;
3758                         off += sizeof(struct frag_hdr);
3759                         break;
3760                 }
3761                 default:
3762                         done = true;
3763                         break;
3764                 }
3765         }
3766 
3767         err = -EPROTO;
3768 
3769         if (!done || fragment)
3770                 goto out;
3771 
3772         csum = skb_checksum_setup_ip(skb, nexthdr, off);
3773         if (IS_ERR(csum))
3774                 return PTR_ERR(csum);
3775 
3776         if (recalculate)
3777                 *csum = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3778                                          &ipv6_hdr(skb)->daddr,
3779                                          skb->len - off, nexthdr, 0);
3780         err = 0;
3781 
3782 out:
3783         return err;
3784 }
3785 
3786 /**
3787  * skb_checksum_setup - set up partial checksum offset
3788  * @skb: the skb to set up
3789  * @recalculate: if true the pseudo-header checksum will be recalculated
3790  */
3791 int skb_checksum_setup(struct sk_buff *skb, bool recalculate)
3792 {
3793         int err;
3794 
3795         switch (skb->protocol) {
3796         case htons(ETH_P_IP):
3797                 err = skb_checksum_setup_ipv4(skb, recalculate);
3798                 break;
3799 
3800         case htons(ETH_P_IPV6):
3801                 err = skb_checksum_setup_ipv6(skb, recalculate);
3802                 break;
3803 
3804         default:
3805                 err = -EPROTO;
3806                 break;
3807         }
3808 
3809         return err;
3810 }
3811 EXPORT_SYMBOL(skb_checksum_setup);
3812 
3813 void __skb_warn_lro_forwarding(const struct sk_buff *skb)
3814 {
3815         net_warn_ratelimited("%s: received packets cannot be forwarded while LRO is enabled\n",
3816                              skb->dev->name);
3817 }
3818 EXPORT_SYMBOL(__skb_warn_lro_forwarding);
3819 
3820 void kfree_skb_partial(struct sk_buff *skb, bool head_stolen)
3821 {
3822         if (head_stolen) {
3823                 skb_release_head_state(skb);
3824                 kmem_cache_free(skbuff_head_cache, skb);
3825         } else {
3826                 __kfree_skb(skb);
3827         }
3828 }
3829 EXPORT_SYMBOL(kfree_skb_partial);
3830 
3831 /**
3832  * skb_try_coalesce - try to merge skb to prior one
3833  * @to: prior buffer
3834  * @from: buffer to add
3835  * @fragstolen: pointer to boolean
3836  * @delta_truesize: how much more was allocated than was requested
3837  */
3838 bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
3839                       bool *fragstolen, int *delta_truesize)
3840 {
3841         int i, delta, len = from->len;
3842 
3843         *fragstolen = false;
3844 
3845         if (skb_cloned(to))
3846                 return false;
3847 
3848         if (len <= skb_tailroom(to)) {
3849                 BUG_ON(skb_copy_bits(from, 0, skb_put(to, len), len));
3850                 *delta_truesize = 0;
3851                 return true;
3852         }
3853 
3854         if (skb_has_frag_list(to) || skb_has_frag_list(from))
3855                 return false;
3856 
3857         if (skb_headlen(from) != 0) {
3858                 struct page *page;
3859                 unsigned int offset;
3860 
3861                 if (skb_shinfo(to)->nr_frags +
3862                     skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS)
3863                         return false;
3864 
3865                 if (skb_head_is_locked(from))
3866                         return false;
3867 
3868                 delta = from->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff));
3869 
3870                 page = virt_to_head_page(from->head);
3871                 offset = from->data - (unsigned char *)page_address(page);
3872 
3873                 skb_fill_page_desc(to, skb_shinfo(to)->nr_frags,
3874                                    page, offset, skb_headlen(from));
3875                 *fragstolen = true;
3876         } else {
3877                 if (skb_shinfo(to)->nr_frags +
3878                     skb_shinfo(from)->nr_frags > MAX_SKB_FRAGS)
3879                         return false;
3880 
3881                 delta = from->truesize - SKB_TRUESIZE(skb_end_offset(from));
3882         }
3883 
3884         WARN_ON_ONCE(delta < len);
3885 
3886         memcpy(skb_shinfo(to)->frags + skb_shinfo(to)->nr_frags,
3887                skb_shinfo(from)->frags,
3888                skb_shinfo(from)->nr_frags * sizeof(skb_frag_t));
3889         skb_shinfo(to)->nr_frags += skb_shinfo(from)->nr_frags;
3890 
3891         if (!skb_cloned(from))
3892                 skb_shinfo(from)->nr_frags = 0;
3893 
3894         /* if the skb is not cloned this does nothing
3895          * since we set nr_frags to 0.
3896          */
3897         for (i = 0; i < skb_shinfo(from)->nr_frags; i++)
3898                 skb_frag_ref(from, i);
3899 
3900         to->truesize += delta;
3901         to->len += len;
3902         to->data_len += len;
3903 
3904         *delta_truesize = delta;
3905         return true;
3906 }
3907 EXPORT_SYMBOL(skb_try_coalesce);
3908 
3909 /**
3910  * skb_scrub_packet - scrub an skb
3911  *
3912  * @skb: buffer to clean
3913  * @xnet: packet is crossing netns
3914  *
3915  * skb_scrub_packet can be used after encapsulating or decapsulting a packet
3916  * into/from a tunnel. Some information have to be cleared during these
3917  * operations.
3918  * skb_scrub_packet can also be used to clean a skb before injecting it in
3919  * another namespace (@xnet == true). We have to clear all information in the
3920  * skb that could impact namespace isolation.
3921  */
3922 void skb_scrub_packet(struct sk_buff *skb, bool xnet)
3923 {
3924         if (xnet)
3925                 skb_orphan(skb);
3926         skb->tstamp.tv64 = 0;
3927         skb->pkt_type = PACKET_HOST;
3928         skb->skb_iif = 0;
3929         skb->ignore_df = 0;
3930         skb_dst_drop(skb);
3931         skb->mark = 0;
3932         secpath_reset(skb);
3933         nf_reset(skb);
3934         nf_reset_trace(skb);
3935 }
3936 EXPORT_SYMBOL_GPL(skb_scrub_packet);
3937 
3938 /**
3939  * skb_gso_transport_seglen - Return length of individual segments of a gso packet
3940  *
3941  * @skb: GSO skb
3942  *
3943  * skb_gso_transport_seglen is used to determine the real size of the
3944  * individual segments, including Layer4 headers (TCP/UDP).
3945  *
3946  * The MAC/L2 or network (IP, IPv6) headers are not accounted for.
3947  */
3948 unsigned int skb_gso_transport_seglen(const struct sk_buff *skb)
3949 {
3950         const struct skb_shared_info *shinfo = skb_shinfo(skb);
3951 
3952         if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
3953                 return tcp_hdrlen(skb) + shinfo->gso_size;
3954 
3955         /* UFO sets gso_size to the size of the fragmentation
3956          * payload, i.e. the size of the L4 (UDP) header is already
3957          * accounted for.
3958          */
3959         return shinfo->gso_size;
3960 }
3961 EXPORT_SYMBOL_GPL(skb_gso_transport_seglen);
3962 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us