Version:  2.0.40 2.2.26 2.4.37 3.3 3.4 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16 3.17 3.18 3.19

Linux/include/linux/skbuff.h

  1 /*
  2  *      Definitions for the 'struct sk_buff' memory handlers.
  3  *
  4  *      Authors:
  5  *              Alan Cox, <gw4pts@gw4pts.ampr.org>
  6  *              Florian La Roche, <rzsfl@rz.uni-sb.de>
  7  *
  8  *      This program is free software; you can redistribute it and/or
  9  *      modify it under the terms of the GNU General Public License
 10  *      as published by the Free Software Foundation; either version
 11  *      2 of the License, or (at your option) any later version.
 12  */
 13 
 14 #ifndef _LINUX_SKBUFF_H
 15 #define _LINUX_SKBUFF_H
 16 
 17 #include <linux/kernel.h>
 18 #include <linux/kmemcheck.h>
 19 #include <linux/compiler.h>
 20 #include <linux/time.h>
 21 #include <linux/bug.h>
 22 #include <linux/cache.h>
 23 #include <linux/rbtree.h>
 24 #include <linux/socket.h>
 25 
 26 #include <linux/atomic.h>
 27 #include <asm/types.h>
 28 #include <linux/spinlock.h>
 29 #include <linux/net.h>
 30 #include <linux/textsearch.h>
 31 #include <net/checksum.h>
 32 #include <linux/rcupdate.h>
 33 #include <linux/hrtimer.h>
 34 #include <linux/dma-mapping.h>
 35 #include <linux/netdev_features.h>
 36 #include <linux/sched.h>
 37 #include <net/flow_keys.h>
 38 
 39 /* A. Checksumming of received packets by device.
 40  *
 41  * CHECKSUM_NONE:
 42  *
 43  *   Device failed to checksum this packet e.g. due to lack of capabilities.
 44  *   The packet contains full (though not verified) checksum in packet but
 45  *   not in skb->csum. Thus, skb->csum is undefined in this case.
 46  *
 47  * CHECKSUM_UNNECESSARY:
 48  *
 49  *   The hardware you're dealing with doesn't calculate the full checksum
 50  *   (as in CHECKSUM_COMPLETE), but it does parse headers and verify checksums
 51  *   for specific protocols. For such packets it will set CHECKSUM_UNNECESSARY
 52  *   if their checksums are okay. skb->csum is still undefined in this case
 53  *   though. It is a bad option, but, unfortunately, nowadays most vendors do
 54  *   this. Apparently with the secret goal to sell you new devices, when you
 55  *   will add new protocol to your host, f.e. IPv6 8)
 56  *
 57  *   CHECKSUM_UNNECESSARY is applicable to following protocols:
 58  *     TCP: IPv6 and IPv4.
 59  *     UDP: IPv4 and IPv6. A device may apply CHECKSUM_UNNECESSARY to a
 60  *       zero UDP checksum for either IPv4 or IPv6, the networking stack
 61  *       may perform further validation in this case.
 62  *     GRE: only if the checksum is present in the header.
 63  *     SCTP: indicates the CRC in SCTP header has been validated.
 64  *
 65  *   skb->csum_level indicates the number of consecutive checksums found in
 66  *   the packet minus one that have been verified as CHECKSUM_UNNECESSARY.
 67  *   For instance if a device receives an IPv6->UDP->GRE->IPv4->TCP packet
 68  *   and a device is able to verify the checksums for UDP (possibly zero),
 69  *   GRE (checksum flag is set), and TCP-- skb->csum_level would be set to
 70  *   two. If the device were only able to verify the UDP checksum and not
 71  *   GRE, either because it doesn't support GRE checksum of because GRE
 72  *   checksum is bad, skb->csum_level would be set to zero (TCP checksum is
 73  *   not considered in this case).
 74  *
 75  * CHECKSUM_COMPLETE:
 76  *
 77  *   This is the most generic way. The device supplied checksum of the _whole_
 78  *   packet as seen by netif_rx() and fills out in skb->csum. Meaning, the
 79  *   hardware doesn't need to parse L3/L4 headers to implement this.
 80  *
 81  *   Note: Even if device supports only some protocols, but is able to produce
 82  *   skb->csum, it MUST use CHECKSUM_COMPLETE, not CHECKSUM_UNNECESSARY.
 83  *
 84  * CHECKSUM_PARTIAL:
 85  *
 86  *   This is identical to the case for output below. This may occur on a packet
 87  *   received directly from another Linux OS, e.g., a virtualized Linux kernel
 88  *   on the same host. The packet can be treated in the same way as
 89  *   CHECKSUM_UNNECESSARY, except that on output (i.e., forwarding) the
 90  *   checksum must be filled in by the OS or the hardware.
 91  *
 92  * B. Checksumming on output.
 93  *
 94  * CHECKSUM_NONE:
 95  *
 96  *   The skb was already checksummed by the protocol, or a checksum is not
 97  *   required.
 98  *
 99  * CHECKSUM_PARTIAL:
100  *
101  *   The device is required to checksum the packet as seen by hard_start_xmit()
102  *   from skb->csum_start up to the end, and to record/write the checksum at
103  *   offset skb->csum_start + skb->csum_offset.
104  *
105  *   The device must show its capabilities in dev->features, set up at device
106  *   setup time, e.g. netdev_features.h:
107  *
108  *      NETIF_F_HW_CSUM - It's a clever device, it's able to checksum everything.
109  *      NETIF_F_IP_CSUM - Device is dumb, it's able to checksum only TCP/UDP over
110  *                        IPv4. Sigh. Vendors like this way for an unknown reason.
111  *                        Though, see comment above about CHECKSUM_UNNECESSARY. 8)
112  *      NETIF_F_IPV6_CSUM - About as dumb as the last one but does IPv6 instead.
113  *      NETIF_F_...     - Well, you get the picture.
114  *
115  * CHECKSUM_UNNECESSARY:
116  *
117  *   Normally, the device will do per protocol specific checksumming. Protocol
118  *   implementations that do not want the NIC to perform the checksum
119  *   calculation should use this flag in their outgoing skbs.
120  *
121  *      NETIF_F_FCOE_CRC - This indicates that the device can do FCoE FC CRC
122  *                         offload. Correspondingly, the FCoE protocol driver
123  *                         stack should use CHECKSUM_UNNECESSARY.
124  *
125  * Any questions? No questions, good.           --ANK
126  */
127 
128 /* Don't change this without changing skb_csum_unnecessary! */
129 #define CHECKSUM_NONE           0
130 #define CHECKSUM_UNNECESSARY    1
131 #define CHECKSUM_COMPLETE       2
132 #define CHECKSUM_PARTIAL        3
133 
134 /* Maximum value in skb->csum_level */
135 #define SKB_MAX_CSUM_LEVEL      3
136 
137 #define SKB_DATA_ALIGN(X)       ALIGN(X, SMP_CACHE_BYTES)
138 #define SKB_WITH_OVERHEAD(X)    \
139         ((X) - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
140 #define SKB_MAX_ORDER(X, ORDER) \
141         SKB_WITH_OVERHEAD((PAGE_SIZE << (ORDER)) - (X))
142 #define SKB_MAX_HEAD(X)         (SKB_MAX_ORDER((X), 0))
143 #define SKB_MAX_ALLOC           (SKB_MAX_ORDER(0, 2))
144 
145 /* return minimum truesize of one skb containing X bytes of data */
146 #define SKB_TRUESIZE(X) ((X) +                                          \
147                          SKB_DATA_ALIGN(sizeof(struct sk_buff)) +       \
148                          SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
149 
150 struct net_device;
151 struct scatterlist;
152 struct pipe_inode_info;
153 struct iov_iter;
154 struct napi_struct;
155 
156 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
157 struct nf_conntrack {
158         atomic_t use;
159 };
160 #endif
161 
162 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
163 struct nf_bridge_info {
164         atomic_t                use;
165         unsigned int            mask;
166         struct net_device       *physindev;
167         struct net_device       *physoutdev;
168         unsigned long           data[32 / sizeof(unsigned long)];
169 };
170 #endif
171 
172 struct sk_buff_head {
173         /* These two members must be first. */
174         struct sk_buff  *next;
175         struct sk_buff  *prev;
176 
177         __u32           qlen;
178         spinlock_t      lock;
179 };
180 
181 struct sk_buff;
182 
183 /* To allow 64K frame to be packed as single skb without frag_list we
184  * require 64K/PAGE_SIZE pages plus 1 additional page to allow for
185  * buffers which do not start on a page boundary.
186  *
187  * Since GRO uses frags we allocate at least 16 regardless of page
188  * size.
189  */
190 #if (65536/PAGE_SIZE + 1) < 16
191 #define MAX_SKB_FRAGS 16UL
192 #else
193 #define MAX_SKB_FRAGS (65536/PAGE_SIZE + 1)
194 #endif
195 
196 typedef struct skb_frag_struct skb_frag_t;
197 
198 struct skb_frag_struct {
199         struct {
200                 struct page *p;
201         } page;
202 #if (BITS_PER_LONG > 32) || (PAGE_SIZE >= 65536)
203         __u32 page_offset;
204         __u32 size;
205 #else
206         __u16 page_offset;
207         __u16 size;
208 #endif
209 };
210 
211 static inline unsigned int skb_frag_size(const skb_frag_t *frag)
212 {
213         return frag->size;
214 }
215 
216 static inline void skb_frag_size_set(skb_frag_t *frag, unsigned int size)
217 {
218         frag->size = size;
219 }
220 
221 static inline void skb_frag_size_add(skb_frag_t *frag, int delta)
222 {
223         frag->size += delta;
224 }
225 
226 static inline void skb_frag_size_sub(skb_frag_t *frag, int delta)
227 {
228         frag->size -= delta;
229 }
230 
231 #define HAVE_HW_TIME_STAMP
232 
233 /**
234  * struct skb_shared_hwtstamps - hardware time stamps
235  * @hwtstamp:   hardware time stamp transformed into duration
236  *              since arbitrary point in time
237  *
238  * Software time stamps generated by ktime_get_real() are stored in
239  * skb->tstamp.
240  *
241  * hwtstamps can only be compared against other hwtstamps from
242  * the same device.
243  *
244  * This structure is attached to packets as part of the
245  * &skb_shared_info. Use skb_hwtstamps() to get a pointer.
246  */
247 struct skb_shared_hwtstamps {
248         ktime_t hwtstamp;
249 };
250 
251 /* Definitions for tx_flags in struct skb_shared_info */
252 enum {
253         /* generate hardware time stamp */
254         SKBTX_HW_TSTAMP = 1 << 0,
255 
256         /* generate software time stamp when queueing packet to NIC */
257         SKBTX_SW_TSTAMP = 1 << 1,
258 
259         /* device driver is going to provide hardware time stamp */
260         SKBTX_IN_PROGRESS = 1 << 2,
261 
262         /* device driver supports TX zero-copy buffers */
263         SKBTX_DEV_ZEROCOPY = 1 << 3,
264 
265         /* generate wifi status information (where possible) */
266         SKBTX_WIFI_STATUS = 1 << 4,
267 
268         /* This indicates at least one fragment might be overwritten
269          * (as in vmsplice(), sendfile() ...)
270          * If we need to compute a TX checksum, we'll need to copy
271          * all frags to avoid possible bad checksum
272          */
273         SKBTX_SHARED_FRAG = 1 << 5,
274 
275         /* generate software time stamp when entering packet scheduling */
276         SKBTX_SCHED_TSTAMP = 1 << 6,
277 
278         /* generate software timestamp on peer data acknowledgment */
279         SKBTX_ACK_TSTAMP = 1 << 7,
280 };
281 
282 #define SKBTX_ANY_SW_TSTAMP     (SKBTX_SW_TSTAMP    | \
283                                  SKBTX_SCHED_TSTAMP | \
284                                  SKBTX_ACK_TSTAMP)
285 #define SKBTX_ANY_TSTAMP        (SKBTX_HW_TSTAMP | SKBTX_ANY_SW_TSTAMP)
286 
287 /*
288  * The callback notifies userspace to release buffers when skb DMA is done in
289  * lower device, the skb last reference should be 0 when calling this.
290  * The zerocopy_success argument is true if zero copy transmit occurred,
291  * false on data copy or out of memory error caused by data copy attempt.
292  * The ctx field is used to track device context.
293  * The desc field is used to track userspace buffer index.
294  */
295 struct ubuf_info {
296         void (*callback)(struct ubuf_info *, bool zerocopy_success);
297         void *ctx;
298         unsigned long desc;
299 };
300 
301 /* This data is invariant across clones and lives at
302  * the end of the header data, ie. at skb->end.
303  */
304 struct skb_shared_info {
305         unsigned char   nr_frags;
306         __u8            tx_flags;
307         unsigned short  gso_size;
308         /* Warning: this field is not always filled in (UFO)! */
309         unsigned short  gso_segs;
310         unsigned short  gso_type;
311         struct sk_buff  *frag_list;
312         struct skb_shared_hwtstamps hwtstamps;
313         u32             tskey;
314         __be32          ip6_frag_id;
315 
316         /*
317          * Warning : all fields before dataref are cleared in __alloc_skb()
318          */
319         atomic_t        dataref;
320 
321         /* Intermediate layers must ensure that destructor_arg
322          * remains valid until skb destructor */
323         void *          destructor_arg;
324 
325         /* must be last field, see pskb_expand_head() */
326         skb_frag_t      frags[MAX_SKB_FRAGS];
327 };
328 
329 /* We divide dataref into two halves.  The higher 16 bits hold references
330  * to the payload part of skb->data.  The lower 16 bits hold references to
331  * the entire skb->data.  A clone of a headerless skb holds the length of
332  * the header in skb->hdr_len.
333  *
334  * All users must obey the rule that the skb->data reference count must be
335  * greater than or equal to the payload reference count.
336  *
337  * Holding a reference to the payload part means that the user does not
338  * care about modifications to the header part of skb->data.
339  */
340 #define SKB_DATAREF_SHIFT 16
341 #define SKB_DATAREF_MASK ((1 << SKB_DATAREF_SHIFT) - 1)
342 
343 
344 enum {
345         SKB_FCLONE_UNAVAILABLE, /* skb has no fclone (from head_cache) */
346         SKB_FCLONE_ORIG,        /* orig skb (from fclone_cache) */
347         SKB_FCLONE_CLONE,       /* companion fclone skb (from fclone_cache) */
348 };
349 
350 enum {
351         SKB_GSO_TCPV4 = 1 << 0,
352         SKB_GSO_UDP = 1 << 1,
353 
354         /* This indicates the skb is from an untrusted source. */
355         SKB_GSO_DODGY = 1 << 2,
356 
357         /* This indicates the tcp segment has CWR set. */
358         SKB_GSO_TCP_ECN = 1 << 3,
359 
360         SKB_GSO_TCPV6 = 1 << 4,
361 
362         SKB_GSO_FCOE = 1 << 5,
363 
364         SKB_GSO_GRE = 1 << 6,
365 
366         SKB_GSO_GRE_CSUM = 1 << 7,
367 
368         SKB_GSO_IPIP = 1 << 8,
369 
370         SKB_GSO_SIT = 1 << 9,
371 
372         SKB_GSO_UDP_TUNNEL = 1 << 10,
373 
374         SKB_GSO_UDP_TUNNEL_CSUM = 1 << 11,
375 
376         SKB_GSO_TUNNEL_REMCSUM = 1 << 12,
377 };
378 
379 #if BITS_PER_LONG > 32
380 #define NET_SKBUFF_DATA_USES_OFFSET 1
381 #endif
382 
383 #ifdef NET_SKBUFF_DATA_USES_OFFSET
384 typedef unsigned int sk_buff_data_t;
385 #else
386 typedef unsigned char *sk_buff_data_t;
387 #endif
388 
389 /**
390  * struct skb_mstamp - multi resolution time stamps
391  * @stamp_us: timestamp in us resolution
392  * @stamp_jiffies: timestamp in jiffies
393  */
394 struct skb_mstamp {
395         union {
396                 u64             v64;
397                 struct {
398                         u32     stamp_us;
399                         u32     stamp_jiffies;
400                 };
401         };
402 };
403 
404 /**
405  * skb_mstamp_get - get current timestamp
406  * @cl: place to store timestamps
407  */
408 static inline void skb_mstamp_get(struct skb_mstamp *cl)
409 {
410         u64 val = local_clock();
411 
412         do_div(val, NSEC_PER_USEC);
413         cl->stamp_us = (u32)val;
414         cl->stamp_jiffies = (u32)jiffies;
415 }
416 
417 /**
418  * skb_mstamp_delta - compute the difference in usec between two skb_mstamp
419  * @t1: pointer to newest sample
420  * @t0: pointer to oldest sample
421  */
422 static inline u32 skb_mstamp_us_delta(const struct skb_mstamp *t1,
423                                       const struct skb_mstamp *t0)
424 {
425         s32 delta_us = t1->stamp_us - t0->stamp_us;
426         u32 delta_jiffies = t1->stamp_jiffies - t0->stamp_jiffies;
427 
428         /* If delta_us is negative, this might be because interval is too big,
429          * or local_clock() drift is too big : fallback using jiffies.
430          */
431         if (delta_us <= 0 ||
432             delta_jiffies >= (INT_MAX / (USEC_PER_SEC / HZ)))
433 
434                 delta_us = jiffies_to_usecs(delta_jiffies);
435 
436         return delta_us;
437 }
438 
439 
440 /** 
441  *      struct sk_buff - socket buffer
442  *      @next: Next buffer in list
443  *      @prev: Previous buffer in list
444  *      @tstamp: Time we arrived/left
445  *      @rbnode: RB tree node, alternative to next/prev for netem/tcp
446  *      @sk: Socket we are owned by
447  *      @dev: Device we arrived on/are leaving by
448  *      @cb: Control buffer. Free for use by every layer. Put private vars here
449  *      @_skb_refdst: destination entry (with norefcount bit)
450  *      @sp: the security path, used for xfrm
451  *      @len: Length of actual data
452  *      @data_len: Data length
453  *      @mac_len: Length of link layer header
454  *      @hdr_len: writable header length of cloned skb
455  *      @csum: Checksum (must include start/offset pair)
456  *      @csum_start: Offset from skb->head where checksumming should start
457  *      @csum_offset: Offset from csum_start where checksum should be stored
458  *      @priority: Packet queueing priority
459  *      @ignore_df: allow local fragmentation
460  *      @cloned: Head may be cloned (check refcnt to be sure)
461  *      @ip_summed: Driver fed us an IP checksum
462  *      @nohdr: Payload reference only, must not modify header
463  *      @nfctinfo: Relationship of this skb to the connection
464  *      @pkt_type: Packet class
465  *      @fclone: skbuff clone status
466  *      @ipvs_property: skbuff is owned by ipvs
467  *      @peeked: this packet has been seen already, so stats have been
468  *              done for it, don't do them again
469  *      @nf_trace: netfilter packet trace flag
470  *      @protocol: Packet protocol from driver
471  *      @destructor: Destruct function
472  *      @nfct: Associated connection, if any
473  *      @nf_bridge: Saved data about a bridged frame - see br_netfilter.c
474  *      @skb_iif: ifindex of device we arrived on
475  *      @tc_index: Traffic control index
476  *      @tc_verd: traffic control verdict
477  *      @hash: the packet hash
478  *      @queue_mapping: Queue mapping for multiqueue devices
479  *      @xmit_more: More SKBs are pending for this queue
480  *      @ndisc_nodetype: router type (from link layer)
481  *      @ooo_okay: allow the mapping of a socket to a queue to be changed
482  *      @l4_hash: indicate hash is a canonical 4-tuple hash over transport
483  *              ports.
484  *      @sw_hash: indicates hash was computed in software stack
485  *      @wifi_acked_valid: wifi_acked was set
486  *      @wifi_acked: whether frame was acked on wifi or not
487  *      @no_fcs:  Request NIC to treat last 4 bytes as Ethernet FCS
488   *     @napi_id: id of the NAPI struct this skb came from
489  *      @secmark: security marking
490  *      @mark: Generic packet mark
491  *      @dropcount: total number of sk_receive_queue overflows
492  *      @vlan_proto: vlan encapsulation protocol
493  *      @vlan_tci: vlan tag control information
494  *      @inner_protocol: Protocol (encapsulation)
495  *      @inner_transport_header: Inner transport layer header (encapsulation)
496  *      @inner_network_header: Network layer header (encapsulation)
497  *      @inner_mac_header: Link layer header (encapsulation)
498  *      @transport_header: Transport layer header
499  *      @network_header: Network layer header
500  *      @mac_header: Link layer header
501  *      @tail: Tail pointer
502  *      @end: End pointer
503  *      @head: Head of buffer
504  *      @data: Data head pointer
505  *      @truesize: Buffer size
506  *      @users: User count - see {datagram,tcp}.c
507  */
508 
509 struct sk_buff {
510         union {
511                 struct {
512                         /* These two members must be first. */
513                         struct sk_buff          *next;
514                         struct sk_buff          *prev;
515 
516                         union {
517                                 ktime_t         tstamp;
518                                 struct skb_mstamp skb_mstamp;
519                         };
520                 };
521                 struct rb_node  rbnode; /* used in netem & tcp stack */
522         };
523         struct sock             *sk;
524         struct net_device       *dev;
525 
526         /*
527          * This is the control buffer. It is free to use for every
528          * layer. Please put your private variables there. If you
529          * want to keep them across layers you have to do a skb_clone()
530          * first. This is owned by whoever has the skb queued ATM.
531          */
532         char                    cb[48] __aligned(8);
533 
534         unsigned long           _skb_refdst;
535         void                    (*destructor)(struct sk_buff *skb);
536 #ifdef CONFIG_XFRM
537         struct  sec_path        *sp;
538 #endif
539 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
540         struct nf_conntrack     *nfct;
541 #endif
542 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
543         struct nf_bridge_info   *nf_bridge;
544 #endif
545         unsigned int            len,
546                                 data_len;
547         __u16                   mac_len,
548                                 hdr_len;
549 
550         /* Following fields are _not_ copied in __copy_skb_header()
551          * Note that queue_mapping is here mostly to fill a hole.
552          */
553         kmemcheck_bitfield_begin(flags1);
554         __u16                   queue_mapping;
555         __u8                    cloned:1,
556                                 nohdr:1,
557                                 fclone:2,
558                                 peeked:1,
559                                 head_frag:1,
560                                 xmit_more:1;
561         /* one bit hole */
562         kmemcheck_bitfield_end(flags1);
563 
564         /* fields enclosed in headers_start/headers_end are copied
565          * using a single memcpy() in __copy_skb_header()
566          */
567         /* private: */
568         __u32                   headers_start[0];
569         /* public: */
570 
571 /* if you move pkt_type around you also must adapt those constants */
572 #ifdef __BIG_ENDIAN_BITFIELD
573 #define PKT_TYPE_MAX    (7 << 5)
574 #else
575 #define PKT_TYPE_MAX    7
576 #endif
577 #define PKT_TYPE_OFFSET()       offsetof(struct sk_buff, __pkt_type_offset)
578 
579         __u8                    __pkt_type_offset[0];
580         __u8                    pkt_type:3;
581         __u8                    pfmemalloc:1;
582         __u8                    ignore_df:1;
583         __u8                    nfctinfo:3;
584 
585         __u8                    nf_trace:1;
586         __u8                    ip_summed:2;
587         __u8                    ooo_okay:1;
588         __u8                    l4_hash:1;
589         __u8                    sw_hash:1;
590         __u8                    wifi_acked_valid:1;
591         __u8                    wifi_acked:1;
592 
593         __u8                    no_fcs:1;
594         /* Indicates the inner headers are valid in the skbuff. */
595         __u8                    encapsulation:1;
596         __u8                    encap_hdr_csum:1;
597         __u8                    csum_valid:1;
598         __u8                    csum_complete_sw:1;
599         __u8                    csum_level:2;
600         __u8                    csum_bad:1;
601 
602 #ifdef CONFIG_IPV6_NDISC_NODETYPE
603         __u8                    ndisc_nodetype:2;
604 #endif
605         __u8                    ipvs_property:1;
606         __u8                    inner_protocol_type:1;
607         __u8                    remcsum_offload:1;
608         /* 3 or 5 bit hole */
609 
610 #ifdef CONFIG_NET_SCHED
611         __u16                   tc_index;       /* traffic control index */
612 #ifdef CONFIG_NET_CLS_ACT
613         __u16                   tc_verd;        /* traffic control verdict */
614 #endif
615 #endif
616 
617         union {
618                 __wsum          csum;
619                 struct {
620                         __u16   csum_start;
621                         __u16   csum_offset;
622                 };
623         };
624         __u32                   priority;
625         int                     skb_iif;
626         __u32                   hash;
627         __be16                  vlan_proto;
628         __u16                   vlan_tci;
629 #ifdef CONFIG_NET_RX_BUSY_POLL
630         unsigned int    napi_id;
631 #endif
632 #ifdef CONFIG_NETWORK_SECMARK
633         __u32                   secmark;
634 #endif
635         union {
636                 __u32           mark;
637                 __u32           dropcount;
638                 __u32           reserved_tailroom;
639         };
640 
641         union {
642                 __be16          inner_protocol;
643                 __u8            inner_ipproto;
644         };
645 
646         __u16                   inner_transport_header;
647         __u16                   inner_network_header;
648         __u16                   inner_mac_header;
649 
650         __be16                  protocol;
651         __u16                   transport_header;
652         __u16                   network_header;
653         __u16                   mac_header;
654 
655         /* private: */
656         __u32                   headers_end[0];
657         /* public: */
658 
659         /* These elements must be at the end, see alloc_skb() for details.  */
660         sk_buff_data_t          tail;
661         sk_buff_data_t          end;
662         unsigned char           *head,
663                                 *data;
664         unsigned int            truesize;
665         atomic_t                users;
666 };
667 
668 #ifdef __KERNEL__
669 /*
670  *      Handling routines are only of interest to the kernel
671  */
672 #include <linux/slab.h>
673 
674 
675 #define SKB_ALLOC_FCLONE        0x01
676 #define SKB_ALLOC_RX            0x02
677 #define SKB_ALLOC_NAPI          0x04
678 
679 /* Returns true if the skb was allocated from PFMEMALLOC reserves */
680 static inline bool skb_pfmemalloc(const struct sk_buff *skb)
681 {
682         return unlikely(skb->pfmemalloc);
683 }
684 
685 /*
686  * skb might have a dst pointer attached, refcounted or not.
687  * _skb_refdst low order bit is set if refcount was _not_ taken
688  */
689 #define SKB_DST_NOREF   1UL
690 #define SKB_DST_PTRMASK ~(SKB_DST_NOREF)
691 
692 /**
693  * skb_dst - returns skb dst_entry
694  * @skb: buffer
695  *
696  * Returns skb dst_entry, regardless of reference taken or not.
697  */
698 static inline struct dst_entry *skb_dst(const struct sk_buff *skb)
699 {
700         /* If refdst was not refcounted, check we still are in a 
701          * rcu_read_lock section
702          */
703         WARN_ON((skb->_skb_refdst & SKB_DST_NOREF) &&
704                 !rcu_read_lock_held() &&
705                 !rcu_read_lock_bh_held());
706         return (struct dst_entry *)(skb->_skb_refdst & SKB_DST_PTRMASK);
707 }
708 
709 /**
710  * skb_dst_set - sets skb dst
711  * @skb: buffer
712  * @dst: dst entry
713  *
714  * Sets skb dst, assuming a reference was taken on dst and should
715  * be released by skb_dst_drop()
716  */
717 static inline void skb_dst_set(struct sk_buff *skb, struct dst_entry *dst)
718 {
719         skb->_skb_refdst = (unsigned long)dst;
720 }
721 
722 /**
723  * skb_dst_set_noref - sets skb dst, hopefully, without taking reference
724  * @skb: buffer
725  * @dst: dst entry
726  *
727  * Sets skb dst, assuming a reference was not taken on dst.
728  * If dst entry is cached, we do not take reference and dst_release
729  * will be avoided by refdst_drop. If dst entry is not cached, we take
730  * reference, so that last dst_release can destroy the dst immediately.
731  */
732 static inline void skb_dst_set_noref(struct sk_buff *skb, struct dst_entry *dst)
733 {
734         WARN_ON(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
735         skb->_skb_refdst = (unsigned long)dst | SKB_DST_NOREF;
736 }
737 
738 /**
739  * skb_dst_is_noref - Test if skb dst isn't refcounted
740  * @skb: buffer
741  */
742 static inline bool skb_dst_is_noref(const struct sk_buff *skb)
743 {
744         return (skb->_skb_refdst & SKB_DST_NOREF) && skb_dst(skb);
745 }
746 
747 static inline struct rtable *skb_rtable(const struct sk_buff *skb)
748 {
749         return (struct rtable *)skb_dst(skb);
750 }
751 
752 void kfree_skb(struct sk_buff *skb);
753 void kfree_skb_list(struct sk_buff *segs);
754 void skb_tx_error(struct sk_buff *skb);
755 void consume_skb(struct sk_buff *skb);
756 void  __kfree_skb(struct sk_buff *skb);
757 extern struct kmem_cache *skbuff_head_cache;
758 
759 void kfree_skb_partial(struct sk_buff *skb, bool head_stolen);
760 bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
761                       bool *fragstolen, int *delta_truesize);
762 
763 struct sk_buff *__alloc_skb(unsigned int size, gfp_t priority, int flags,
764                             int node);
765 struct sk_buff *build_skb(void *data, unsigned int frag_size);
766 static inline struct sk_buff *alloc_skb(unsigned int size,
767                                         gfp_t priority)
768 {
769         return __alloc_skb(size, priority, 0, NUMA_NO_NODE);
770 }
771 
772 struct sk_buff *alloc_skb_with_frags(unsigned long header_len,
773                                      unsigned long data_len,
774                                      int max_page_order,
775                                      int *errcode,
776                                      gfp_t gfp_mask);
777 
778 /* Layout of fast clones : [skb1][skb2][fclone_ref] */
779 struct sk_buff_fclones {
780         struct sk_buff  skb1;
781 
782         struct sk_buff  skb2;
783 
784         atomic_t        fclone_ref;
785 };
786 
787 /**
788  *      skb_fclone_busy - check if fclone is busy
789  *      @skb: buffer
790  *
791  * Returns true is skb is a fast clone, and its clone is not freed.
792  * Some drivers call skb_orphan() in their ndo_start_xmit(),
793  * so we also check that this didnt happen.
794  */
795 static inline bool skb_fclone_busy(const struct sock *sk,
796                                    const struct sk_buff *skb)
797 {
798         const struct sk_buff_fclones *fclones;
799 
800         fclones = container_of(skb, struct sk_buff_fclones, skb1);
801 
802         return skb->fclone == SKB_FCLONE_ORIG &&
803                atomic_read(&fclones->fclone_ref) > 1 &&
804                fclones->skb2.sk == sk;
805 }
806 
807 static inline struct sk_buff *alloc_skb_fclone(unsigned int size,
808                                                gfp_t priority)
809 {
810         return __alloc_skb(size, priority, SKB_ALLOC_FCLONE, NUMA_NO_NODE);
811 }
812 
813 struct sk_buff *__alloc_skb_head(gfp_t priority, int node);
814 static inline struct sk_buff *alloc_skb_head(gfp_t priority)
815 {
816         return __alloc_skb_head(priority, -1);
817 }
818 
819 struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src);
820 int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask);
821 struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t priority);
822 struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t priority);
823 struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom,
824                                    gfp_t gfp_mask, bool fclone);
825 static inline struct sk_buff *__pskb_copy(struct sk_buff *skb, int headroom,
826                                           gfp_t gfp_mask)
827 {
828         return __pskb_copy_fclone(skb, headroom, gfp_mask, false);
829 }
830 
831 int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, gfp_t gfp_mask);
832 struct sk_buff *skb_realloc_headroom(struct sk_buff *skb,
833                                      unsigned int headroom);
834 struct sk_buff *skb_copy_expand(const struct sk_buff *skb, int newheadroom,
835                                 int newtailroom, gfp_t priority);
836 int skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg,
837                         int offset, int len);
838 int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset,
839                  int len);
840 int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer);
841 int skb_pad(struct sk_buff *skb, int pad);
842 #define dev_kfree_skb(a)        consume_skb(a)
843 
844 int skb_append_datato_frags(struct sock *sk, struct sk_buff *skb,
845                             int getfrag(void *from, char *to, int offset,
846                                         int len, int odd, struct sk_buff *skb),
847                             void *from, int length);
848 
849 struct skb_seq_state {
850         __u32           lower_offset;
851         __u32           upper_offset;
852         __u32           frag_idx;
853         __u32           stepped_offset;
854         struct sk_buff  *root_skb;
855         struct sk_buff  *cur_skb;
856         __u8            *frag_data;
857 };
858 
859 void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
860                           unsigned int to, struct skb_seq_state *st);
861 unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
862                           struct skb_seq_state *st);
863 void skb_abort_seq_read(struct skb_seq_state *st);
864 
865 unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
866                            unsigned int to, struct ts_config *config,
867                            struct ts_state *state);
868 
869 /*
870  * Packet hash types specify the type of hash in skb_set_hash.
871  *
872  * Hash types refer to the protocol layer addresses which are used to
873  * construct a packet's hash. The hashes are used to differentiate or identify
874  * flows of the protocol layer for the hash type. Hash types are either
875  * layer-2 (L2), layer-3 (L3), or layer-4 (L4).
876  *
877  * Properties of hashes:
878  *
879  * 1) Two packets in different flows have different hash values
880  * 2) Two packets in the same flow should have the same hash value
881  *
882  * A hash at a higher layer is considered to be more specific. A driver should
883  * set the most specific hash possible.
884  *
885  * A driver cannot indicate a more specific hash than the layer at which a hash
886  * was computed. For instance an L3 hash cannot be set as an L4 hash.
887  *
888  * A driver may indicate a hash level which is less specific than the
889  * actual layer the hash was computed on. For instance, a hash computed
890  * at L4 may be considered an L3 hash. This should only be done if the
891  * driver can't unambiguously determine that the HW computed the hash at
892  * the higher layer. Note that the "should" in the second property above
893  * permits this.
894  */
895 enum pkt_hash_types {
896         PKT_HASH_TYPE_NONE,     /* Undefined type */
897         PKT_HASH_TYPE_L2,       /* Input: src_MAC, dest_MAC */
898         PKT_HASH_TYPE_L3,       /* Input: src_IP, dst_IP */
899         PKT_HASH_TYPE_L4,       /* Input: src_IP, dst_IP, src_port, dst_port */
900 };
901 
902 static inline void
903 skb_set_hash(struct sk_buff *skb, __u32 hash, enum pkt_hash_types type)
904 {
905         skb->l4_hash = (type == PKT_HASH_TYPE_L4);
906         skb->sw_hash = 0;
907         skb->hash = hash;
908 }
909 
910 void __skb_get_hash(struct sk_buff *skb);
911 static inline __u32 skb_get_hash(struct sk_buff *skb)
912 {
913         if (!skb->l4_hash && !skb->sw_hash)
914                 __skb_get_hash(skb);
915 
916         return skb->hash;
917 }
918 
919 static inline __u32 skb_get_hash_raw(const struct sk_buff *skb)
920 {
921         return skb->hash;
922 }
923 
924 static inline void skb_clear_hash(struct sk_buff *skb)
925 {
926         skb->hash = 0;
927         skb->sw_hash = 0;
928         skb->l4_hash = 0;
929 }
930 
931 static inline void skb_clear_hash_if_not_l4(struct sk_buff *skb)
932 {
933         if (!skb->l4_hash)
934                 skb_clear_hash(skb);
935 }
936 
937 static inline void skb_copy_hash(struct sk_buff *to, const struct sk_buff *from)
938 {
939         to->hash = from->hash;
940         to->sw_hash = from->sw_hash;
941         to->l4_hash = from->l4_hash;
942 };
943 
944 #ifdef NET_SKBUFF_DATA_USES_OFFSET
945 static inline unsigned char *skb_end_pointer(const struct sk_buff *skb)
946 {
947         return skb->head + skb->end;
948 }
949 
950 static inline unsigned int skb_end_offset(const struct sk_buff *skb)
951 {
952         return skb->end;
953 }
954 #else
955 static inline unsigned char *skb_end_pointer(const struct sk_buff *skb)
956 {
957         return skb->end;
958 }
959 
960 static inline unsigned int skb_end_offset(const struct sk_buff *skb)
961 {
962         return skb->end - skb->head;
963 }
964 #endif
965 
966 /* Internal */
967 #define skb_shinfo(SKB) ((struct skb_shared_info *)(skb_end_pointer(SKB)))
968 
969 static inline struct skb_shared_hwtstamps *skb_hwtstamps(struct sk_buff *skb)
970 {
971         return &skb_shinfo(skb)->hwtstamps;
972 }
973 
974 /**
975  *      skb_queue_empty - check if a queue is empty
976  *      @list: queue head
977  *
978  *      Returns true if the queue is empty, false otherwise.
979  */
980 static inline int skb_queue_empty(const struct sk_buff_head *list)
981 {
982         return list->next == (const struct sk_buff *) list;
983 }
984 
985 /**
986  *      skb_queue_is_last - check if skb is the last entry in the queue
987  *      @list: queue head
988  *      @skb: buffer
989  *
990  *      Returns true if @skb is the last buffer on the list.
991  */
992 static inline bool skb_queue_is_last(const struct sk_buff_head *list,
993                                      const struct sk_buff *skb)
994 {
995         return skb->next == (const struct sk_buff *) list;
996 }
997 
998 /**
999  *      skb_queue_is_first - check if skb is the first entry in the queue
1000  *      @list: queue head
1001  *      @skb: buffer
1002  *
1003  *      Returns true if @skb is the first buffer on the list.
1004  */
1005 static inline bool skb_queue_is_first(const struct sk_buff_head *list,
1006                                       const struct sk_buff *skb)
1007 {
1008         return skb->prev == (const struct sk_buff *) list;
1009 }
1010 
1011 /**
1012  *      skb_queue_next - return the next packet in the queue
1013  *      @list: queue head
1014  *      @skb: current buffer
1015  *
1016  *      Return the next packet in @list after @skb.  It is only valid to
1017  *      call this if skb_queue_is_last() evaluates to false.
1018  */
1019 static inline struct sk_buff *skb_queue_next(const struct sk_buff_head *list,
1020                                              const struct sk_buff *skb)
1021 {
1022         /* This BUG_ON may seem severe, but if we just return then we
1023          * are going to dereference garbage.
1024          */
1025         BUG_ON(skb_queue_is_last(list, skb));
1026         return skb->next;
1027 }
1028 
1029 /**
1030  *      skb_queue_prev - return the prev packet in the queue
1031  *      @list: queue head
1032  *      @skb: current buffer
1033  *
1034  *      Return the prev packet in @list before @skb.  It is only valid to
1035  *      call this if skb_queue_is_first() evaluates to false.
1036  */
1037 static inline struct sk_buff *skb_queue_prev(const struct sk_buff_head *list,
1038                                              const struct sk_buff *skb)
1039 {
1040         /* This BUG_ON may seem severe, but if we just return then we
1041          * are going to dereference garbage.
1042          */
1043         BUG_ON(skb_queue_is_first(list, skb));
1044         return skb->prev;
1045 }
1046 
1047 /**
1048  *      skb_get - reference buffer
1049  *      @skb: buffer to reference
1050  *
1051  *      Makes another reference to a socket buffer and returns a pointer
1052  *      to the buffer.
1053  */
1054 static inline struct sk_buff *skb_get(struct sk_buff *skb)
1055 {
1056         atomic_inc(&skb->users);
1057         return skb;
1058 }
1059 
1060 /*
1061  * If users == 1, we are the only owner and are can avoid redundant
1062  * atomic change.
1063  */
1064 
1065 /**
1066  *      skb_cloned - is the buffer a clone
1067  *      @skb: buffer to check
1068  *
1069  *      Returns true if the buffer was generated with skb_clone() and is
1070  *      one of multiple shared copies of the buffer. Cloned buffers are
1071  *      shared data so must not be written to under normal circumstances.
1072  */
1073 static inline int skb_cloned(const struct sk_buff *skb)
1074 {
1075         return skb->cloned &&
1076                (atomic_read(&skb_shinfo(skb)->dataref) & SKB_DATAREF_MASK) != 1;
1077 }
1078 
1079 static inline int skb_unclone(struct sk_buff *skb, gfp_t pri)
1080 {
1081         might_sleep_if(pri & __GFP_WAIT);
1082 
1083         if (skb_cloned(skb))
1084                 return pskb_expand_head(skb, 0, 0, pri);
1085 
1086         return 0;
1087 }
1088 
1089 /**
1090  *      skb_header_cloned - is the header a clone
1091  *      @skb: buffer to check
1092  *
1093  *      Returns true if modifying the header part of the buffer requires
1094  *      the data to be copied.
1095  */
1096 static inline int skb_header_cloned(const struct sk_buff *skb)
1097 {
1098         int dataref;
1099 
1100         if (!skb->cloned)
1101                 return 0;
1102 
1103         dataref = atomic_read(&skb_shinfo(skb)->dataref);
1104         dataref = (dataref & SKB_DATAREF_MASK) - (dataref >> SKB_DATAREF_SHIFT);
1105         return dataref != 1;
1106 }
1107 
1108 /**
1109  *      skb_header_release - release reference to header
1110  *      @skb: buffer to operate on
1111  *
1112  *      Drop a reference to the header part of the buffer.  This is done
1113  *      by acquiring a payload reference.  You must not read from the header
1114  *      part of skb->data after this.
1115  *      Note : Check if you can use __skb_header_release() instead.
1116  */
1117 static inline void skb_header_release(struct sk_buff *skb)
1118 {
1119         BUG_ON(skb->nohdr);
1120         skb->nohdr = 1;
1121         atomic_add(1 << SKB_DATAREF_SHIFT, &skb_shinfo(skb)->dataref);
1122 }
1123 
1124 /**
1125  *      __skb_header_release - release reference to header
1126  *      @skb: buffer to operate on
1127  *
1128  *      Variant of skb_header_release() assuming skb is private to caller.
1129  *      We can avoid one atomic operation.
1130  */
1131 static inline void __skb_header_release(struct sk_buff *skb)
1132 {
1133         skb->nohdr = 1;
1134         atomic_set(&skb_shinfo(skb)->dataref, 1 + (1 << SKB_DATAREF_SHIFT));
1135 }
1136 
1137 
1138 /**
1139  *      skb_shared - is the buffer shared
1140  *      @skb: buffer to check
1141  *
1142  *      Returns true if more than one person has a reference to this
1143  *      buffer.
1144  */
1145 static inline int skb_shared(const struct sk_buff *skb)
1146 {
1147         return atomic_read(&skb->users) != 1;
1148 }
1149 
1150 /**
1151  *      skb_share_check - check if buffer is shared and if so clone it
1152  *      @skb: buffer to check
1153  *      @pri: priority for memory allocation
1154  *
1155  *      If the buffer is shared the buffer is cloned and the old copy
1156  *      drops a reference. A new clone with a single reference is returned.
1157  *      If the buffer is not shared the original buffer is returned. When
1158  *      being called from interrupt status or with spinlocks held pri must
1159  *      be GFP_ATOMIC.
1160  *
1161  *      NULL is returned on a memory allocation failure.
1162  */
1163 static inline struct sk_buff *skb_share_check(struct sk_buff *skb, gfp_t pri)
1164 {
1165         might_sleep_if(pri & __GFP_WAIT);
1166         if (skb_shared(skb)) {
1167                 struct sk_buff *nskb = skb_clone(skb, pri);
1168 
1169                 if (likely(nskb))
1170                         consume_skb(skb);
1171                 else
1172                         kfree_skb(skb);
1173                 skb = nskb;
1174         }
1175         return skb;
1176 }
1177 
1178 /*
1179  *      Copy shared buffers into a new sk_buff. We effectively do COW on
1180  *      packets to handle cases where we have a local reader and forward
1181  *      and a couple of other messy ones. The normal one is tcpdumping
1182  *      a packet thats being forwarded.
1183  */
1184 
1185 /**
1186  *      skb_unshare - make a copy of a shared buffer
1187  *      @skb: buffer to check
1188  *      @pri: priority for memory allocation
1189  *
1190  *      If the socket buffer is a clone then this function creates a new
1191  *      copy of the data, drops a reference count on the old copy and returns
1192  *      the new copy with the reference count at 1. If the buffer is not a clone
1193  *      the original buffer is returned. When called with a spinlock held or
1194  *      from interrupt state @pri must be %GFP_ATOMIC
1195  *
1196  *      %NULL is returned on a memory allocation failure.
1197  */
1198 static inline struct sk_buff *skb_unshare(struct sk_buff *skb,
1199                                           gfp_t pri)
1200 {
1201         might_sleep_if(pri & __GFP_WAIT);
1202         if (skb_cloned(skb)) {
1203                 struct sk_buff *nskb = skb_copy(skb, pri);
1204 
1205                 /* Free our shared copy */
1206                 if (likely(nskb))
1207                         consume_skb(skb);
1208                 else
1209                         kfree_skb(skb);
1210                 skb = nskb;
1211         }
1212         return skb;
1213 }
1214 
1215 /**
1216  *      skb_peek - peek at the head of an &sk_buff_head
1217  *      @list_: list to peek at
1218  *
1219  *      Peek an &sk_buff. Unlike most other operations you _MUST_
1220  *      be careful with this one. A peek leaves the buffer on the
1221  *      list and someone else may run off with it. You must hold
1222  *      the appropriate locks or have a private queue to do this.
1223  *
1224  *      Returns %NULL for an empty list or a pointer to the head element.
1225  *      The reference count is not incremented and the reference is therefore
1226  *      volatile. Use with caution.
1227  */
1228 static inline struct sk_buff *skb_peek(const struct sk_buff_head *list_)
1229 {
1230         struct sk_buff *skb = list_->next;
1231 
1232         if (skb == (struct sk_buff *)list_)
1233                 skb = NULL;
1234         return skb;
1235 }
1236 
1237 /**
1238  *      skb_peek_next - peek skb following the given one from a queue
1239  *      @skb: skb to start from
1240  *      @list_: list to peek at
1241  *
1242  *      Returns %NULL when the end of the list is met or a pointer to the
1243  *      next element. The reference count is not incremented and the
1244  *      reference is therefore volatile. Use with caution.
1245  */
1246 static inline struct sk_buff *skb_peek_next(struct sk_buff *skb,
1247                 const struct sk_buff_head *list_)
1248 {
1249         struct sk_buff *next = skb->next;
1250 
1251         if (next == (struct sk_buff *)list_)
1252                 next = NULL;
1253         return next;
1254 }
1255 
1256 /**
1257  *      skb_peek_tail - peek at the tail of an &sk_buff_head
1258  *      @list_: list to peek at
1259  *
1260  *      Peek an &sk_buff. Unlike most other operations you _MUST_
1261  *      be careful with this one. A peek leaves the buffer on the
1262  *      list and someone else may run off with it. You must hold
1263  *      the appropriate locks or have a private queue to do this.
1264  *
1265  *      Returns %NULL for an empty list or a pointer to the tail element.
1266  *      The reference count is not incremented and the reference is therefore
1267  *      volatile. Use with caution.
1268  */
1269 static inline struct sk_buff *skb_peek_tail(const struct sk_buff_head *list_)
1270 {
1271         struct sk_buff *skb = list_->prev;
1272 
1273         if (skb == (struct sk_buff *)list_)
1274                 skb = NULL;
1275         return skb;
1276 
1277 }
1278 
1279 /**
1280  *      skb_queue_len   - get queue length
1281  *      @list_: list to measure
1282  *
1283  *      Return the length of an &sk_buff queue.
1284  */
1285 static inline __u32 skb_queue_len(const struct sk_buff_head *list_)
1286 {
1287         return list_->qlen;
1288 }
1289 
1290 /**
1291  *      __skb_queue_head_init - initialize non-spinlock portions of sk_buff_head
1292  *      @list: queue to initialize
1293  *
1294  *      This initializes only the list and queue length aspects of
1295  *      an sk_buff_head object.  This allows to initialize the list
1296  *      aspects of an sk_buff_head without reinitializing things like
1297  *      the spinlock.  It can also be used for on-stack sk_buff_head
1298  *      objects where the spinlock is known to not be used.
1299  */
1300 static inline void __skb_queue_head_init(struct sk_buff_head *list)
1301 {
1302         list->prev = list->next = (struct sk_buff *)list;
1303         list->qlen = 0;
1304 }
1305 
1306 /*
1307  * This function creates a split out lock class for each invocation;
1308  * this is needed for now since a whole lot of users of the skb-queue
1309  * infrastructure in drivers have different locking usage (in hardirq)
1310  * than the networking core (in softirq only). In the long run either the
1311  * network layer or drivers should need annotation to consolidate the
1312  * main types of usage into 3 classes.
1313  */
1314 static inline void skb_queue_head_init(struct sk_buff_head *list)
1315 {
1316         spin_lock_init(&list->lock);
1317         __skb_queue_head_init(list);
1318 }
1319 
1320 static inline void skb_queue_head_init_class(struct sk_buff_head *list,
1321                 struct lock_class_key *class)
1322 {
1323         skb_queue_head_init(list);
1324         lockdep_set_class(&list->lock, class);
1325 }
1326 
1327 /*
1328  *      Insert an sk_buff on a list.
1329  *
1330  *      The "__skb_xxxx()" functions are the non-atomic ones that
1331  *      can only be called with interrupts disabled.
1332  */
1333 void skb_insert(struct sk_buff *old, struct sk_buff *newsk,
1334                 struct sk_buff_head *list);
1335 static inline void __skb_insert(struct sk_buff *newsk,
1336                                 struct sk_buff *prev, struct sk_buff *next,
1337                                 struct sk_buff_head *list)
1338 {
1339         newsk->next = next;
1340         newsk->prev = prev;
1341         next->prev  = prev->next = newsk;
1342         list->qlen++;
1343 }
1344 
1345 static inline void __skb_queue_splice(const struct sk_buff_head *list,
1346                                       struct sk_buff *prev,
1347                                       struct sk_buff *next)
1348 {
1349         struct sk_buff *first = list->next;
1350         struct sk_buff *last = list->prev;
1351 
1352         first->prev = prev;
1353         prev->next = first;
1354 
1355         last->next = next;
1356         next->prev = last;
1357 }
1358 
1359 /**
1360  *      skb_queue_splice - join two skb lists, this is designed for stacks
1361  *      @list: the new list to add
1362  *      @head: the place to add it in the first list
1363  */
1364 static inline void skb_queue_splice(const struct sk_buff_head *list,
1365                                     struct sk_buff_head *head)
1366 {
1367         if (!skb_queue_empty(list)) {
1368                 __skb_queue_splice(list, (struct sk_buff *) head, head->next);
1369                 head->qlen += list->qlen;
1370         }
1371 }
1372 
1373 /**
1374  *      skb_queue_splice_init - join two skb lists and reinitialise the emptied list
1375  *      @list: the new list to add
1376  *      @head: the place to add it in the first list
1377  *
1378  *      The list at @list is reinitialised
1379  */
1380 static inline void skb_queue_splice_init(struct sk_buff_head *list,
1381                                          struct sk_buff_head *head)
1382 {
1383         if (!skb_queue_empty(list)) {
1384                 __skb_queue_splice(list, (struct sk_buff *) head, head->next);
1385                 head->qlen += list->qlen;
1386                 __skb_queue_head_init(list);
1387         }
1388 }
1389 
1390 /**
1391  *      skb_queue_splice_tail - join two skb lists, each list being a queue
1392  *      @list: the new list to add
1393  *      @head: the place to add it in the first list
1394  */
1395 static inline void skb_queue_splice_tail(const struct sk_buff_head *list,
1396                                          struct sk_buff_head *head)
1397 {
1398         if (!skb_queue_empty(list)) {
1399                 __skb_queue_splice(list, head->prev, (struct sk_buff *) head);
1400                 head->qlen += list->qlen;
1401         }
1402 }
1403 
1404 /**
1405  *      skb_queue_splice_tail_init - join two skb lists and reinitialise the emptied list
1406  *      @list: the new list to add
1407  *      @head: the place to add it in the first list
1408  *
1409  *      Each of the lists is a queue.
1410  *      The list at @list is reinitialised
1411  */
1412 static inline void skb_queue_splice_tail_init(struct sk_buff_head *list,
1413                                               struct sk_buff_head *head)
1414 {
1415         if (!skb_queue_empty(list)) {
1416                 __skb_queue_splice(list, head->prev, (struct sk_buff *) head);
1417                 head->qlen += list->qlen;
1418                 __skb_queue_head_init(list);
1419         }
1420 }
1421 
1422 /**
1423  *      __skb_queue_after - queue a buffer at the list head
1424  *      @list: list to use
1425  *      @prev: place after this buffer
1426  *      @newsk: buffer to queue
1427  *
1428  *      Queue a buffer int the middle of a list. This function takes no locks
1429  *      and you must therefore hold required locks before calling it.
1430  *
1431  *      A buffer cannot be placed on two lists at the same time.
1432  */
1433 static inline void __skb_queue_after(struct sk_buff_head *list,
1434                                      struct sk_buff *prev,
1435                                      struct sk_buff *newsk)
1436 {
1437         __skb_insert(newsk, prev, prev->next, list);
1438 }
1439 
1440 void skb_append(struct sk_buff *old, struct sk_buff *newsk,
1441                 struct sk_buff_head *list);
1442 
1443 static inline void __skb_queue_before(struct sk_buff_head *list,
1444                                       struct sk_buff *next,
1445                                       struct sk_buff *newsk)
1446 {
1447         __skb_insert(newsk, next->prev, next, list);
1448 }
1449 
1450 /**
1451  *      __skb_queue_head - queue a buffer at the list head
1452  *      @list: list to use
1453  *      @newsk: buffer to queue
1454  *
1455  *      Queue a buffer at the start of a list. This function takes no locks
1456  *      and you must therefore hold required locks before calling it.
1457  *
1458  *      A buffer cannot be placed on two lists at the same time.
1459  */
1460 void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk);
1461 static inline void __skb_queue_head(struct sk_buff_head *list,
1462                                     struct sk_buff *newsk)
1463 {
1464         __skb_queue_after(list, (struct sk_buff *)list, newsk);
1465 }
1466 
1467 /**
1468  *      __skb_queue_tail - queue a buffer at the list tail
1469  *      @list: list to use
1470  *      @newsk: buffer to queue
1471  *
1472  *      Queue a buffer at the end of a list. This function takes no locks
1473  *      and you must therefore hold required locks before calling it.
1474  *
1475  *      A buffer cannot be placed on two lists at the same time.
1476  */
1477 void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk);
1478 static inline void __skb_queue_tail(struct sk_buff_head *list,
1479                                    struct sk_buff *newsk)
1480 {
1481         __skb_queue_before(list, (struct sk_buff *)list, newsk);
1482 }
1483 
1484 /*
1485  * remove sk_buff from list. _Must_ be called atomically, and with
1486  * the list known..
1487  */
1488 void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list);
1489 static inline void __skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
1490 {
1491         struct sk_buff *next, *prev;
1492 
1493         list->qlen--;
1494         next       = skb->next;
1495         prev       = skb->prev;
1496         skb->next  = skb->prev = NULL;
1497         next->prev = prev;
1498         prev->next = next;
1499 }
1500 
1501 /**
1502  *      __skb_dequeue - remove from the head of the queue
1503  *      @list: list to dequeue from
1504  *
1505  *      Remove the head of the list. This function does not take any locks
1506  *      so must be used with appropriate locks held only. The head item is
1507  *      returned or %NULL if the list is empty.
1508  */
1509 struct sk_buff *skb_dequeue(struct sk_buff_head *list);
1510 static inline struct sk_buff *__skb_dequeue(struct sk_buff_head *list)
1511 {
1512         struct sk_buff *skb = skb_peek(list);
1513         if (skb)
1514                 __skb_unlink(skb, list);
1515         return skb;
1516 }
1517 
1518 /**
1519  *      __skb_dequeue_tail - remove from the tail of the queue
1520  *      @list: list to dequeue from
1521  *
1522  *      Remove the tail of the list. This function does not take any locks
1523  *      so must be used with appropriate locks held only. The tail item is
1524  *      returned or %NULL if the list is empty.
1525  */
1526 struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list);
1527 static inline struct sk_buff *__skb_dequeue_tail(struct sk_buff_head *list)
1528 {
1529         struct sk_buff *skb = skb_peek_tail(list);
1530         if (skb)
1531                 __skb_unlink(skb, list);
1532         return skb;
1533 }
1534 
1535 
1536 static inline bool skb_is_nonlinear(const struct sk_buff *skb)
1537 {
1538         return skb->data_len;
1539 }
1540 
1541 static inline unsigned int skb_headlen(const struct sk_buff *skb)
1542 {
1543         return skb->len - skb->data_len;
1544 }
1545 
1546 static inline int skb_pagelen(const struct sk_buff *skb)
1547 {
1548         int i, len = 0;
1549 
1550         for (i = (int)skb_shinfo(skb)->nr_frags - 1; i >= 0; i--)
1551                 len += skb_frag_size(&skb_shinfo(skb)->frags[i]);
1552         return len + skb_headlen(skb);
1553 }
1554 
1555 /**
1556  * __skb_fill_page_desc - initialise a paged fragment in an skb
1557  * @skb: buffer containing fragment to be initialised
1558  * @i: paged fragment index to initialise
1559  * @page: the page to use for this fragment
1560  * @off: the offset to the data with @page
1561  * @size: the length of the data
1562  *
1563  * Initialises the @i'th fragment of @skb to point to &size bytes at
1564  * offset @off within @page.
1565  *
1566  * Does not take any additional reference on the fragment.
1567  */
1568 static inline void __skb_fill_page_desc(struct sk_buff *skb, int i,
1569                                         struct page *page, int off, int size)
1570 {
1571         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1572 
1573         /*
1574          * Propagate page->pfmemalloc to the skb if we can. The problem is
1575          * that not all callers have unique ownership of the page. If
1576          * pfmemalloc is set, we check the mapping as a mapping implies
1577          * page->index is set (index and pfmemalloc share space).
1578          * If it's a valid mapping, we cannot use page->pfmemalloc but we
1579          * do not lose pfmemalloc information as the pages would not be
1580          * allocated using __GFP_MEMALLOC.
1581          */
1582         frag->page.p              = page;
1583         frag->page_offset         = off;
1584         skb_frag_size_set(frag, size);
1585 
1586         page = compound_head(page);
1587         if (page->pfmemalloc && !page->mapping)
1588                 skb->pfmemalloc = true;
1589 }
1590 
1591 /**
1592  * skb_fill_page_desc - initialise a paged fragment in an skb
1593  * @skb: buffer containing fragment to be initialised
1594  * @i: paged fragment index to initialise
1595  * @page: the page to use for this fragment
1596  * @off: the offset to the data with @page
1597  * @size: the length of the data
1598  *
1599  * As per __skb_fill_page_desc() -- initialises the @i'th fragment of
1600  * @skb to point to @size bytes at offset @off within @page. In
1601  * addition updates @skb such that @i is the last fragment.
1602  *
1603  * Does not take any additional reference on the fragment.
1604  */
1605 static inline void skb_fill_page_desc(struct sk_buff *skb, int i,
1606                                       struct page *page, int off, int size)
1607 {
1608         __skb_fill_page_desc(skb, i, page, off, size);
1609         skb_shinfo(skb)->nr_frags = i + 1;
1610 }
1611 
1612 void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off,
1613                      int size, unsigned int truesize);
1614 
1615 void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size,
1616                           unsigned int truesize);
1617 
1618 #define SKB_PAGE_ASSERT(skb)    BUG_ON(skb_shinfo(skb)->nr_frags)
1619 #define SKB_FRAG_ASSERT(skb)    BUG_ON(skb_has_frag_list(skb))
1620 #define SKB_LINEAR_ASSERT(skb)  BUG_ON(skb_is_nonlinear(skb))
1621 
1622 #ifdef NET_SKBUFF_DATA_USES_OFFSET
1623 static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb)
1624 {
1625         return skb->head + skb->tail;
1626 }
1627 
1628 static inline void skb_reset_tail_pointer(struct sk_buff *skb)
1629 {
1630         skb->tail = skb->data - skb->head;
1631 }
1632 
1633 static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset)
1634 {
1635         skb_reset_tail_pointer(skb);
1636         skb->tail += offset;
1637 }
1638 
1639 #else /* NET_SKBUFF_DATA_USES_OFFSET */
1640 static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb)
1641 {
1642         return skb->tail;
1643 }
1644 
1645 static inline void skb_reset_tail_pointer(struct sk_buff *skb)
1646 {
1647         skb->tail = skb->data;
1648 }
1649 
1650 static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset)
1651 {
1652         skb->tail = skb->data + offset;
1653 }
1654 
1655 #endif /* NET_SKBUFF_DATA_USES_OFFSET */
1656 
1657 /*
1658  *      Add data to an sk_buff
1659  */
1660 unsigned char *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len);
1661 unsigned char *skb_put(struct sk_buff *skb, unsigned int len);
1662 static inline unsigned char *__skb_put(struct sk_buff *skb, unsigned int len)
1663 {
1664         unsigned char *tmp = skb_tail_pointer(skb);
1665         SKB_LINEAR_ASSERT(skb);
1666         skb->tail += len;
1667         skb->len  += len;
1668         return tmp;
1669 }
1670 
1671 unsigned char *skb_push(struct sk_buff *skb, unsigned int len);
1672 static inline unsigned char *__skb_push(struct sk_buff *skb, unsigned int len)
1673 {
1674         skb->data -= len;
1675         skb->len  += len;
1676         return skb->data;
1677 }
1678 
1679 unsigned char *skb_pull(struct sk_buff *skb, unsigned int len);
1680 static inline unsigned char *__skb_pull(struct sk_buff *skb, unsigned int len)
1681 {
1682         skb->len -= len;
1683         BUG_ON(skb->len < skb->data_len);
1684         return skb->data += len;
1685 }
1686 
1687 static inline unsigned char *skb_pull_inline(struct sk_buff *skb, unsigned int len)
1688 {
1689         return unlikely(len > skb->len) ? NULL : __skb_pull(skb, len);
1690 }
1691 
1692 unsigned char *__pskb_pull_tail(struct sk_buff *skb, int delta);
1693 
1694 static inline unsigned char *__pskb_pull(struct sk_buff *skb, unsigned int len)
1695 {
1696         if (len > skb_headlen(skb) &&
1697             !__pskb_pull_tail(skb, len - skb_headlen(skb)))
1698                 return NULL;
1699         skb->len -= len;
1700         return skb->data += len;
1701 }
1702 
1703 static inline unsigned char *pskb_pull(struct sk_buff *skb, unsigned int len)
1704 {
1705         return unlikely(len > skb->len) ? NULL : __pskb_pull(skb, len);
1706 }
1707 
1708 static inline int pskb_may_pull(struct sk_buff *skb, unsigned int len)
1709 {
1710         if (likely(len <= skb_headlen(skb)))
1711                 return 1;
1712         if (unlikely(len > skb->len))
1713                 return 0;
1714         return __pskb_pull_tail(skb, len - skb_headlen(skb)) != NULL;
1715 }
1716 
1717 /**
1718  *      skb_headroom - bytes at buffer head
1719  *      @skb: buffer to check
1720  *
1721  *      Return the number of bytes of free space at the head of an &sk_buff.
1722  */
1723 static inline unsigned int skb_headroom(const struct sk_buff *skb)
1724 {
1725         return skb->data - skb->head;
1726 }
1727 
1728 /**
1729  *      skb_tailroom - bytes at buffer end
1730  *      @skb: buffer to check
1731  *
1732  *      Return the number of bytes of free space at the tail of an sk_buff
1733  */
1734 static inline int skb_tailroom(const struct sk_buff *skb)
1735 {
1736         return skb_is_nonlinear(skb) ? 0 : skb->end - skb->tail;
1737 }
1738 
1739 /**
1740  *      skb_availroom - bytes at buffer end
1741  *      @skb: buffer to check
1742  *
1743  *      Return the number of bytes of free space at the tail of an sk_buff
1744  *      allocated by sk_stream_alloc()
1745  */
1746 static inline int skb_availroom(const struct sk_buff *skb)
1747 {
1748         if (skb_is_nonlinear(skb))
1749                 return 0;
1750 
1751         return skb->end - skb->tail - skb->reserved_tailroom;
1752 }
1753 
1754 /**
1755  *      skb_reserve - adjust headroom
1756  *      @skb: buffer to alter
1757  *      @len: bytes to move
1758  *
1759  *      Increase the headroom of an empty &sk_buff by reducing the tail
1760  *      room. This is only allowed for an empty buffer.
1761  */
1762 static inline void skb_reserve(struct sk_buff *skb, int len)
1763 {
1764         skb->data += len;
1765         skb->tail += len;
1766 }
1767 
1768 #define ENCAP_TYPE_ETHER        0
1769 #define ENCAP_TYPE_IPPROTO      1
1770 
1771 static inline void skb_set_inner_protocol(struct sk_buff *skb,
1772                                           __be16 protocol)
1773 {
1774         skb->inner_protocol = protocol;
1775         skb->inner_protocol_type = ENCAP_TYPE_ETHER;
1776 }
1777 
1778 static inline void skb_set_inner_ipproto(struct sk_buff *skb,
1779                                          __u8 ipproto)
1780 {
1781         skb->inner_ipproto = ipproto;
1782         skb->inner_protocol_type = ENCAP_TYPE_IPPROTO;
1783 }
1784 
1785 static inline void skb_reset_inner_headers(struct sk_buff *skb)
1786 {
1787         skb->inner_mac_header = skb->mac_header;
1788         skb->inner_network_header = skb->network_header;
1789         skb->inner_transport_header = skb->transport_header;
1790 }
1791 
1792 static inline void skb_reset_mac_len(struct sk_buff *skb)
1793 {
1794         skb->mac_len = skb->network_header - skb->mac_header;
1795 }
1796 
1797 static inline unsigned char *skb_inner_transport_header(const struct sk_buff
1798                                                         *skb)
1799 {
1800         return skb->head + skb->inner_transport_header;
1801 }
1802 
1803 static inline void skb_reset_inner_transport_header(struct sk_buff *skb)
1804 {
1805         skb->inner_transport_header = skb->data - skb->head;
1806 }
1807 
1808 static inline void skb_set_inner_transport_header(struct sk_buff *skb,
1809                                                    const int offset)
1810 {
1811         skb_reset_inner_transport_header(skb);
1812         skb->inner_transport_header += offset;
1813 }
1814 
1815 static inline unsigned char *skb_inner_network_header(const struct sk_buff *skb)
1816 {
1817         return skb->head + skb->inner_network_header;
1818 }
1819 
1820 static inline void skb_reset_inner_network_header(struct sk_buff *skb)
1821 {
1822         skb->inner_network_header = skb->data - skb->head;
1823 }
1824 
1825 static inline void skb_set_inner_network_header(struct sk_buff *skb,
1826                                                 const int offset)
1827 {
1828         skb_reset_inner_network_header(skb);
1829         skb->inner_network_header += offset;
1830 }
1831 
1832 static inline unsigned char *skb_inner_mac_header(const struct sk_buff *skb)
1833 {
1834         return skb->head + skb->inner_mac_header;
1835 }
1836 
1837 static inline void skb_reset_inner_mac_header(struct sk_buff *skb)
1838 {
1839         skb->inner_mac_header = skb->data - skb->head;
1840 }
1841 
1842 static inline void skb_set_inner_mac_header(struct sk_buff *skb,
1843                                             const int offset)
1844 {
1845         skb_reset_inner_mac_header(skb);
1846         skb->inner_mac_header += offset;
1847 }
1848 static inline bool skb_transport_header_was_set(const struct sk_buff *skb)
1849 {
1850         return skb->transport_header != (typeof(skb->transport_header))~0U;
1851 }
1852 
1853 static inline unsigned char *skb_transport_header(const struct sk_buff *skb)
1854 {
1855         return skb->head + skb->transport_header;
1856 }
1857 
1858 static inline void skb_reset_transport_header(struct sk_buff *skb)
1859 {
1860         skb->transport_header = skb->data - skb->head;
1861 }
1862 
1863 static inline void skb_set_transport_header(struct sk_buff *skb,
1864                                             const int offset)
1865 {
1866         skb_reset_transport_header(skb);
1867         skb->transport_header += offset;
1868 }
1869 
1870 static inline unsigned char *skb_network_header(const struct sk_buff *skb)
1871 {
1872         return skb->head + skb->network_header;
1873 }
1874 
1875 static inline void skb_reset_network_header(struct sk_buff *skb)
1876 {
1877         skb->network_header = skb->data - skb->head;
1878 }
1879 
1880 static inline void skb_set_network_header(struct sk_buff *skb, const int offset)
1881 {
1882         skb_reset_network_header(skb);
1883         skb->network_header += offset;
1884 }
1885 
1886 static inline unsigned char *skb_mac_header(const struct sk_buff *skb)
1887 {
1888         return skb->head + skb->mac_header;
1889 }
1890 
1891 static inline int skb_mac_header_was_set(const struct sk_buff *skb)
1892 {
1893         return skb->mac_header != (typeof(skb->mac_header))~0U;
1894 }
1895 
1896 static inline void skb_reset_mac_header(struct sk_buff *skb)
1897 {
1898         skb->mac_header = skb->data - skb->head;
1899 }
1900 
1901 static inline void skb_set_mac_header(struct sk_buff *skb, const int offset)
1902 {
1903         skb_reset_mac_header(skb);
1904         skb->mac_header += offset;
1905 }
1906 
1907 static inline void skb_pop_mac_header(struct sk_buff *skb)
1908 {
1909         skb->mac_header = skb->network_header;
1910 }
1911 
1912 static inline void skb_probe_transport_header(struct sk_buff *skb,
1913                                               const int offset_hint)
1914 {
1915         struct flow_keys keys;
1916 
1917         if (skb_transport_header_was_set(skb))
1918                 return;
1919         else if (skb_flow_dissect(skb, &keys))
1920                 skb_set_transport_header(skb, keys.thoff);
1921         else
1922                 skb_set_transport_header(skb, offset_hint);
1923 }
1924 
1925 static inline void skb_mac_header_rebuild(struct sk_buff *skb)
1926 {
1927         if (skb_mac_header_was_set(skb)) {
1928                 const unsigned char *old_mac = skb_mac_header(skb);
1929 
1930                 skb_set_mac_header(skb, -skb->mac_len);
1931                 memmove(skb_mac_header(skb), old_mac, skb->mac_len);
1932         }
1933 }
1934 
1935 static inline int skb_checksum_start_offset(const struct sk_buff *skb)
1936 {
1937         return skb->csum_start - skb_headroom(skb);
1938 }
1939 
1940 static inline int skb_transport_offset(const struct sk_buff *skb)
1941 {
1942         return skb_transport_header(skb) - skb->data;
1943 }
1944 
1945 static inline u32 skb_network_header_len(const struct sk_buff *skb)
1946 {
1947         return skb->transport_header - skb->network_header;
1948 }
1949 
1950 static inline u32 skb_inner_network_header_len(const struct sk_buff *skb)
1951 {
1952         return skb->inner_transport_header - skb->inner_network_header;
1953 }
1954 
1955 static inline int skb_network_offset(const struct sk_buff *skb)
1956 {
1957         return skb_network_header(skb) - skb->data;
1958 }
1959 
1960 static inline int skb_inner_network_offset(const struct sk_buff *skb)
1961 {
1962         return skb_inner_network_header(skb) - skb->data;
1963 }
1964 
1965 static inline int pskb_network_may_pull(struct sk_buff *skb, unsigned int len)
1966 {
1967         return pskb_may_pull(skb, skb_network_offset(skb) + len);
1968 }
1969 
1970 /*
1971  * CPUs often take a performance hit when accessing unaligned memory
1972  * locations. The actual performance hit varies, it can be small if the
1973  * hardware handles it or large if we have to take an exception and fix it
1974  * in software.
1975  *
1976  * Since an ethernet header is 14 bytes network drivers often end up with
1977  * the IP header at an unaligned offset. The IP header can be aligned by
1978  * shifting the start of the packet by 2 bytes. Drivers should do this
1979  * with:
1980  *
1981  * skb_reserve(skb, NET_IP_ALIGN);
1982  *
1983  * The downside to this alignment of the IP header is that the DMA is now
1984  * unaligned. On some architectures the cost of an unaligned DMA is high
1985  * and this cost outweighs the gains made by aligning the IP header.
1986  *
1987  * Since this trade off varies between architectures, we allow NET_IP_ALIGN
1988  * to be overridden.
1989  */
1990 #ifndef NET_IP_ALIGN
1991 #define NET_IP_ALIGN    2
1992 #endif
1993 
1994 /*
1995  * The networking layer reserves some headroom in skb data (via
1996  * dev_alloc_skb). This is used to avoid having to reallocate skb data when
1997  * the header has to grow. In the default case, if the header has to grow
1998  * 32 bytes or less we avoid the reallocation.
1999  *
2000  * Unfortunately this headroom changes the DMA alignment of the resulting
2001  * network packet. As for NET_IP_ALIGN, this unaligned DMA is expensive
2002  * on some architectures. An architecture can override this value,
2003  * perhaps setting it to a cacheline in size (since that will maintain
2004  * cacheline alignment of the DMA). It must be a power of 2.
2005  *
2006  * Various parts of the networking layer expect at least 32 bytes of
2007  * headroom, you should not reduce this.
2008  *
2009  * Using max(32, L1_CACHE_BYTES) makes sense (especially with RPS)
2010  * to reduce average number of cache lines per packet.
2011  * get_rps_cpus() for example only access one 64 bytes aligned block :
2012  * NET_IP_ALIGN(2) + ethernet_header(14) + IP_header(20/40) + ports(8)
2013  */
2014 #ifndef NET_SKB_PAD
2015 #define NET_SKB_PAD     max(32, L1_CACHE_BYTES)
2016 #endif
2017 
2018 int ___pskb_trim(struct sk_buff *skb, unsigned int len);
2019 
2020 static inline void __skb_trim(struct sk_buff *skb, unsigned int len)
2021 {
2022         if (unlikely(skb_is_nonlinear(skb))) {
2023                 WARN_ON(1);
2024                 return;
2025         }
2026         skb->len = len;
2027         skb_set_tail_pointer(skb, len);
2028 }
2029 
2030 void skb_trim(struct sk_buff *skb, unsigned int len);
2031 
2032 static inline int __pskb_trim(struct sk_buff *skb, unsigned int len)
2033 {
2034         if (skb->data_len)
2035                 return ___pskb_trim(skb, len);
2036         __skb_trim(skb, len);
2037         return 0;
2038 }
2039 
2040 static inline int pskb_trim(struct sk_buff *skb, unsigned int len)
2041 {
2042         return (len < skb->len) ? __pskb_trim(skb, len) : 0;
2043 }
2044 
2045 /**
2046  *      pskb_trim_unique - remove end from a paged unique (not cloned) buffer
2047  *      @skb: buffer to alter
2048  *      @len: new length
2049  *
2050  *      This is identical to pskb_trim except that the caller knows that
2051  *      the skb is not cloned so we should never get an error due to out-
2052  *      of-memory.
2053  */
2054 static inline void pskb_trim_unique(struct sk_buff *skb, unsigned int len)
2055 {
2056         int err = pskb_trim(skb, len);
2057         BUG_ON(err);
2058 }
2059 
2060 /**
2061  *      skb_orphan - orphan a buffer
2062  *      @skb: buffer to orphan
2063  *
2064  *      If a buffer currently has an owner then we call the owner's
2065  *      destructor function and make the @skb unowned. The buffer continues
2066  *      to exist but is no longer charged to its former owner.
2067  */
2068 static inline void skb_orphan(struct sk_buff *skb)
2069 {
2070         if (skb->destructor) {
2071                 skb->destructor(skb);
2072                 skb->destructor = NULL;
2073                 skb->sk         = NULL;
2074         } else {
2075                 BUG_ON(skb->sk);
2076         }
2077 }
2078 
2079 /**
2080  *      skb_orphan_frags - orphan the frags contained in a buffer
2081  *      @skb: buffer to orphan frags from
2082  *      @gfp_mask: allocation mask for replacement pages
2083  *
2084  *      For each frag in the SKB which needs a destructor (i.e. has an
2085  *      owner) create a copy of that frag and release the original
2086  *      page by calling the destructor.
2087  */
2088 static inline int skb_orphan_frags(struct sk_buff *skb, gfp_t gfp_mask)
2089 {
2090         if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY)))
2091                 return 0;
2092         return skb_copy_ubufs(skb, gfp_mask);
2093 }
2094 
2095 /**
2096  *      __skb_queue_purge - empty a list
2097  *      @list: list to empty
2098  *
2099  *      Delete all buffers on an &sk_buff list. Each buffer is removed from
2100  *      the list and one reference dropped. This function does not take the
2101  *      list lock and the caller must hold the relevant locks to use it.
2102  */
2103 void skb_queue_purge(struct sk_buff_head *list);
2104 static inline void __skb_queue_purge(struct sk_buff_head *list)
2105 {
2106         struct sk_buff *skb;
2107         while ((skb = __skb_dequeue(list)) != NULL)
2108                 kfree_skb(skb);
2109 }
2110 
2111 #define NETDEV_FRAG_PAGE_MAX_ORDER get_order(32768)
2112 #define NETDEV_FRAG_PAGE_MAX_SIZE  (PAGE_SIZE << NETDEV_FRAG_PAGE_MAX_ORDER)
2113 #define NETDEV_PAGECNT_MAX_BIAS    NETDEV_FRAG_PAGE_MAX_SIZE
2114 
2115 void *netdev_alloc_frag(unsigned int fragsz);
2116 
2117 struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int length,
2118                                    gfp_t gfp_mask);
2119 
2120 /**
2121  *      netdev_alloc_skb - allocate an skbuff for rx on a specific device
2122  *      @dev: network device to receive on
2123  *      @length: length to allocate
2124  *
2125  *      Allocate a new &sk_buff and assign it a usage count of one. The
2126  *      buffer has unspecified headroom built in. Users should allocate
2127  *      the headroom they think they need without accounting for the
2128  *      built in space. The built in space is used for optimisations.
2129  *
2130  *      %NULL is returned if there is no free memory. Although this function
2131  *      allocates memory it can be called from an interrupt.
2132  */
2133 static inline struct sk_buff *netdev_alloc_skb(struct net_device *dev,
2134                                                unsigned int length)
2135 {
2136         return __netdev_alloc_skb(dev, length, GFP_ATOMIC);
2137 }
2138 
2139 /* legacy helper around __netdev_alloc_skb() */
2140 static inline struct sk_buff *__dev_alloc_skb(unsigned int length,
2141                                               gfp_t gfp_mask)
2142 {
2143         return __netdev_alloc_skb(NULL, length, gfp_mask);
2144 }
2145 
2146 /* legacy helper around netdev_alloc_skb() */
2147 static inline struct sk_buff *dev_alloc_skb(unsigned int length)
2148 {
2149         return netdev_alloc_skb(NULL, length);
2150 }
2151 
2152 
2153 static inline struct sk_buff *__netdev_alloc_skb_ip_align(struct net_device *dev,
2154                 unsigned int length, gfp_t gfp)
2155 {
2156         struct sk_buff *skb = __netdev_alloc_skb(dev, length + NET_IP_ALIGN, gfp);
2157 
2158         if (NET_IP_ALIGN && skb)
2159                 skb_reserve(skb, NET_IP_ALIGN);
2160         return skb;
2161 }
2162 
2163 static inline struct sk_buff *netdev_alloc_skb_ip_align(struct net_device *dev,
2164                 unsigned int length)
2165 {
2166         return __netdev_alloc_skb_ip_align(dev, length, GFP_ATOMIC);
2167 }
2168 
2169 void *napi_alloc_frag(unsigned int fragsz);
2170 struct sk_buff *__napi_alloc_skb(struct napi_struct *napi,
2171                                  unsigned int length, gfp_t gfp_mask);
2172 static inline struct sk_buff *napi_alloc_skb(struct napi_struct *napi,
2173                                              unsigned int length)
2174 {
2175         return __napi_alloc_skb(napi, length, GFP_ATOMIC);
2176 }
2177 
2178 /**
2179  * __dev_alloc_pages - allocate page for network Rx
2180  * @gfp_mask: allocation priority. Set __GFP_NOMEMALLOC if not for network Rx
2181  * @order: size of the allocation
2182  *
2183  * Allocate a new page.
2184  *
2185  * %NULL is returned if there is no free memory.
2186 */
2187 static inline struct page *__dev_alloc_pages(gfp_t gfp_mask,
2188                                              unsigned int order)
2189 {
2190         /* This piece of code contains several assumptions.
2191          * 1.  This is for device Rx, therefor a cold page is preferred.
2192          * 2.  The expectation is the user wants a compound page.
2193          * 3.  If requesting a order 0 page it will not be compound
2194          *     due to the check to see if order has a value in prep_new_page
2195          * 4.  __GFP_MEMALLOC is ignored if __GFP_NOMEMALLOC is set due to
2196          *     code in gfp_to_alloc_flags that should be enforcing this.
2197          */
2198         gfp_mask |= __GFP_COLD | __GFP_COMP | __GFP_MEMALLOC;
2199 
2200         return alloc_pages_node(NUMA_NO_NODE, gfp_mask, order);
2201 }
2202 
2203 static inline struct page *dev_alloc_pages(unsigned int order)
2204 {
2205         return __dev_alloc_pages(GFP_ATOMIC, order);
2206 }
2207 
2208 /**
2209  * __dev_alloc_page - allocate a page for network Rx
2210  * @gfp_mask: allocation priority. Set __GFP_NOMEMALLOC if not for network Rx
2211  *
2212  * Allocate a new page.
2213  *
2214  * %NULL is returned if there is no free memory.
2215  */
2216 static inline struct page *__dev_alloc_page(gfp_t gfp_mask)
2217 {
2218         return __dev_alloc_pages(gfp_mask, 0);
2219 }
2220 
2221 static inline struct page *dev_alloc_page(void)
2222 {
2223         return __dev_alloc_page(GFP_ATOMIC);
2224 }
2225 
2226 /**
2227  *      skb_propagate_pfmemalloc - Propagate pfmemalloc if skb is allocated after RX page
2228  *      @page: The page that was allocated from skb_alloc_page
2229  *      @skb: The skb that may need pfmemalloc set
2230  */
2231 static inline void skb_propagate_pfmemalloc(struct page *page,
2232                                              struct sk_buff *skb)
2233 {
2234         if (page && page->pfmemalloc)
2235                 skb->pfmemalloc = true;
2236 }
2237 
2238 /**
2239  * skb_frag_page - retrieve the page referred to by a paged fragment
2240  * @frag: the paged fragment
2241  *
2242  * Returns the &struct page associated with @frag.
2243  */
2244 static inline struct page *skb_frag_page(const skb_frag_t *frag)
2245 {
2246         return frag->page.p;
2247 }
2248 
2249 /**
2250  * __skb_frag_ref - take an addition reference on a paged fragment.
2251  * @frag: the paged fragment
2252  *
2253  * Takes an additional reference on the paged fragment @frag.
2254  */
2255 static inline void __skb_frag_ref(skb_frag_t *frag)
2256 {
2257         get_page(skb_frag_page(frag));
2258 }
2259 
2260 /**
2261  * skb_frag_ref - take an addition reference on a paged fragment of an skb.
2262  * @skb: the buffer
2263  * @f: the fragment offset.
2264  *
2265  * Takes an additional reference on the @f'th paged fragment of @skb.
2266  */
2267 static inline void skb_frag_ref(struct sk_buff *skb, int f)
2268 {
2269         __skb_frag_ref(&skb_shinfo(skb)->frags[f]);
2270 }
2271 
2272 /**
2273  * __skb_frag_unref - release a reference on a paged fragment.
2274  * @frag: the paged fragment
2275  *
2276  * Releases a reference on the paged fragment @frag.
2277  */
2278 static inline void __skb_frag_unref(skb_frag_t *frag)
2279 {
2280         put_page(skb_frag_page(frag));
2281 }
2282 
2283 /**
2284  * skb_frag_unref - release a reference on a paged fragment of an skb.
2285  * @skb: the buffer
2286  * @f: the fragment offset
2287  *
2288  * Releases a reference on the @f'th paged fragment of @skb.
2289  */
2290 static inline void skb_frag_unref(struct sk_buff *skb, int f)
2291 {
2292         __skb_frag_unref(&skb_shinfo(skb)->frags[f]);
2293 }
2294 
2295 /**
2296  * skb_frag_address - gets the address of the data contained in a paged fragment
2297  * @frag: the paged fragment buffer
2298  *
2299  * Returns the address of the data within @frag. The page must already
2300  * be mapped.
2301  */
2302 static inline void *skb_frag_address(const skb_frag_t *frag)
2303 {
2304         return page_address(skb_frag_page(frag)) + frag->page_offset;
2305 }
2306 
2307 /**
2308  * skb_frag_address_safe - gets the address of the data contained in a paged fragment
2309  * @frag: the paged fragment buffer
2310  *
2311  * Returns the address of the data within @frag. Checks that the page
2312  * is mapped and returns %NULL otherwise.
2313  */
2314 static inline void *skb_frag_address_safe(const skb_frag_t *frag)
2315 {
2316         void *ptr = page_address(skb_frag_page(frag));
2317         if (unlikely(!ptr))
2318                 return NULL;
2319 
2320         return ptr + frag->page_offset;
2321 }
2322 
2323 /**
2324  * __skb_frag_set_page - sets the page contained in a paged fragment
2325  * @frag: the paged fragment
2326  * @page: the page to set
2327  *
2328  * Sets the fragment @frag to contain @page.
2329  */
2330 static inline void __skb_frag_set_page(skb_frag_t *frag, struct page *page)
2331 {
2332         frag->page.p = page;
2333 }
2334 
2335 /**
2336  * skb_frag_set_page - sets the page contained in a paged fragment of an skb
2337  * @skb: the buffer
2338  * @f: the fragment offset
2339  * @page: the page to set
2340  *
2341  * Sets the @f'th fragment of @skb to contain @page.
2342  */
2343 static inline void skb_frag_set_page(struct sk_buff *skb, int f,
2344                                      struct page *page)
2345 {
2346         __skb_frag_set_page(&skb_shinfo(skb)->frags[f], page);
2347 }
2348 
2349 bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio);
2350 
2351 /**
2352  * skb_frag_dma_map - maps a paged fragment via the DMA API
2353  * @dev: the device to map the fragment to
2354  * @frag: the paged fragment to map
2355  * @offset: the offset within the fragment (starting at the
2356  *          fragment's own offset)
2357  * @size: the number of bytes to map
2358  * @dir: the direction of the mapping (%PCI_DMA_*)
2359  *
2360  * Maps the page associated with @frag to @device.
2361  */
2362 static inline dma_addr_t skb_frag_dma_map(struct device *dev,
2363                                           const skb_frag_t *frag,
2364                                           size_t offset, size_t size,
2365                                           enum dma_data_direction dir)
2366 {
2367         return dma_map_page(dev, skb_frag_page(frag),
2368                             frag->page_offset + offset, size, dir);
2369 }
2370 
2371 static inline struct sk_buff *pskb_copy(struct sk_buff *skb,
2372                                         gfp_t gfp_mask)
2373 {
2374         return __pskb_copy(skb, skb_headroom(skb), gfp_mask);
2375 }
2376 
2377 
2378 static inline struct sk_buff *pskb_copy_for_clone(struct sk_buff *skb,
2379                                                   gfp_t gfp_mask)
2380 {
2381         return __pskb_copy_fclone(skb, skb_headroom(skb), gfp_mask, true);
2382 }
2383 
2384 
2385 /**
2386  *      skb_clone_writable - is the header of a clone writable
2387  *      @skb: buffer to check
2388  *      @len: length up to which to write
2389  *
2390  *      Returns true if modifying the header part of the cloned buffer
2391  *      does not requires the data to be copied.
2392  */
2393 static inline int skb_clone_writable(const struct sk_buff *skb, unsigned int len)
2394 {
2395         return !skb_header_cloned(skb) &&
2396                skb_headroom(skb) + len <= skb->hdr_len;
2397 }
2398 
2399 static inline int __skb_cow(struct sk_buff *skb, unsigned int headroom,
2400                             int cloned)
2401 {
2402         int delta = 0;
2403 
2404         if (headroom > skb_headroom(skb))
2405                 delta = headroom - skb_headroom(skb);
2406 
2407         if (delta || cloned)
2408                 return pskb_expand_head(skb, ALIGN(delta, NET_SKB_PAD), 0,
2409                                         GFP_ATOMIC);
2410         return 0;
2411 }
2412 
2413 /**
2414  *      skb_cow - copy header of skb when it is required
2415  *      @skb: buffer to cow
2416  *      @headroom: needed headroom
2417  *
2418  *      If the skb passed lacks sufficient headroom or its data part
2419  *      is shared, data is reallocated. If reallocation fails, an error
2420  *      is returned and original skb is not changed.
2421  *
2422  *      The result is skb with writable area skb->head...skb->tail
2423  *      and at least @headroom of space at head.
2424  */
2425 static inline int skb_cow(struct sk_buff *skb, unsigned int headroom)
2426 {
2427         return __skb_cow(skb, headroom, skb_cloned(skb));
2428 }
2429 
2430 /**
2431  *      skb_cow_head - skb_cow but only making the head writable
2432  *      @skb: buffer to cow
2433  *      @headroom: needed headroom
2434  *
2435  *      This function is identical to skb_cow except that we replace the
2436  *      skb_cloned check by skb_header_cloned.  It should be used when
2437  *      you only need to push on some header and do not need to modify
2438  *      the data.
2439  */
2440 static inline int skb_cow_head(struct sk_buff *skb, unsigned int headroom)
2441 {
2442         return __skb_cow(skb, headroom, skb_header_cloned(skb));
2443 }
2444 
2445 /**
2446  *      skb_padto       - pad an skbuff up to a minimal size
2447  *      @skb: buffer to pad
2448  *      @len: minimal length
2449  *
2450  *      Pads up a buffer to ensure the trailing bytes exist and are
2451  *      blanked. If the buffer already contains sufficient data it
2452  *      is untouched. Otherwise it is extended. Returns zero on
2453  *      success. The skb is freed on error.
2454  */
2455 static inline int skb_padto(struct sk_buff *skb, unsigned int len)
2456 {
2457         unsigned int size = skb->len;
2458         if (likely(size >= len))
2459                 return 0;
2460         return skb_pad(skb, len - size);
2461 }
2462 
2463 /**
2464  *      skb_put_padto - increase size and pad an skbuff up to a minimal size
2465  *      @skb: buffer to pad
2466  *      @len: minimal length
2467  *
2468  *      Pads up a buffer to ensure the trailing bytes exist and are
2469  *      blanked. If the buffer already contains sufficient data it
2470  *      is untouched. Otherwise it is extended. Returns zero on
2471  *      success. The skb is freed on error.
2472  */
2473 static inline int skb_put_padto(struct sk_buff *skb, unsigned int len)
2474 {
2475         unsigned int size = skb->len;
2476 
2477         if (unlikely(size < len)) {
2478                 len -= size;
2479                 if (skb_pad(skb, len))
2480                         return -ENOMEM;
2481                 __skb_put(skb, len);
2482         }
2483         return 0;
2484 }
2485 
2486 static inline int skb_add_data(struct sk_buff *skb,
2487                                char __user *from, int copy)
2488 {
2489         const int off = skb->len;
2490 
2491         if (skb->ip_summed == CHECKSUM_NONE) {
2492                 int err = 0;
2493                 __wsum csum = csum_and_copy_from_user(from, skb_put(skb, copy),
2494                                                             copy, 0, &err);
2495                 if (!err) {
2496                         skb->csum = csum_block_add(skb->csum, csum, off);
2497                         return 0;
2498                 }
2499         } else if (!copy_from_user(skb_put(skb, copy), from, copy))
2500                 return 0;
2501 
2502         __skb_trim(skb, off);
2503         return -EFAULT;
2504 }
2505 
2506 static inline bool skb_can_coalesce(struct sk_buff *skb, int i,
2507                                     const struct page *page, int off)
2508 {
2509         if (i) {
2510                 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i - 1];
2511 
2512                 return page == skb_frag_page(frag) &&
2513                        off == frag->page_offset + skb_frag_size(frag);
2514         }
2515         return false;
2516 }
2517 
2518 static inline int __skb_linearize(struct sk_buff *skb)
2519 {
2520         return __pskb_pull_tail(skb, skb->data_len) ? 0 : -ENOMEM;
2521 }
2522 
2523 /**
2524  *      skb_linearize - convert paged skb to linear one
2525  *      @skb: buffer to linarize
2526  *
2527  *      If there is no free memory -ENOMEM is returned, otherwise zero
2528  *      is returned and the old skb data released.
2529  */
2530 static inline int skb_linearize(struct sk_buff *skb)
2531 {
2532         return skb_is_nonlinear(skb) ? __skb_linearize(skb) : 0;
2533 }
2534 
2535 /**
2536  * skb_has_shared_frag - can any frag be overwritten
2537  * @skb: buffer to test
2538  *
2539  * Return true if the skb has at least one frag that might be modified
2540  * by an external entity (as in vmsplice()/sendfile())
2541  */
2542 static inline bool skb_has_shared_frag(const struct sk_buff *skb)
2543 {
2544         return skb_is_nonlinear(skb) &&
2545                skb_shinfo(skb)->tx_flags & SKBTX_SHARED_FRAG;
2546 }
2547 
2548 /**
2549  *      skb_linearize_cow - make sure skb is linear and writable
2550  *      @skb: buffer to process
2551  *
2552  *      If there is no free memory -ENOMEM is returned, otherwise zero
2553  *      is returned and the old skb data released.
2554  */
2555 static inline int skb_linearize_cow(struct sk_buff *skb)
2556 {
2557         return skb_is_nonlinear(skb) || skb_cloned(skb) ?
2558                __skb_linearize(skb) : 0;
2559 }
2560 
2561 /**
2562  *      skb_postpull_rcsum - update checksum for received skb after pull
2563  *      @skb: buffer to update
2564  *      @start: start of data before pull
2565  *      @len: length of data pulled
2566  *
2567  *      After doing a pull on a received packet, you need to call this to
2568  *      update the CHECKSUM_COMPLETE checksum, or set ip_summed to
2569  *      CHECKSUM_NONE so that it can be recomputed from scratch.
2570  */
2571 
2572 static inline void skb_postpull_rcsum(struct sk_buff *skb,
2573                                       const void *start, unsigned int len)
2574 {
2575         if (skb->ip_summed == CHECKSUM_COMPLETE)
2576                 skb->csum = csum_sub(skb->csum, csum_partial(start, len, 0));
2577 }
2578 
2579 unsigned char *skb_pull_rcsum(struct sk_buff *skb, unsigned int len);
2580 
2581 /**
2582  *      pskb_trim_rcsum - trim received skb and update checksum
2583  *      @skb: buffer to trim
2584  *      @len: new length
2585  *
2586  *      This is exactly the same as pskb_trim except that it ensures the
2587  *      checksum of received packets are still valid after the operation.
2588  */
2589 
2590 static inline int pskb_trim_rcsum(struct sk_buff *skb, unsigned int len)
2591 {
2592         if (likely(len >= skb->len))
2593                 return 0;
2594         if (skb->ip_summed == CHECKSUM_COMPLETE)
2595                 skb->ip_summed = CHECKSUM_NONE;
2596         return __pskb_trim(skb, len);
2597 }
2598 
2599 #define skb_queue_walk(queue, skb) \
2600                 for (skb = (queue)->next;                                       \
2601                      skb != (struct sk_buff *)(queue);                          \
2602                      skb = skb->next)
2603 
2604 #define skb_queue_walk_safe(queue, skb, tmp)                                    \
2605                 for (skb = (queue)->next, tmp = skb->next;                      \
2606                      skb != (struct sk_buff *)(queue);                          \
2607                      skb = tmp, tmp = skb->next)
2608 
2609 #define skb_queue_walk_from(queue, skb)                                         \
2610                 for (; skb != (struct sk_buff *)(queue);                        \
2611                      skb = skb->next)
2612 
2613 #define skb_queue_walk_from_safe(queue, skb, tmp)                               \
2614                 for (tmp = skb->next;                                           \
2615                      skb != (struct sk_buff *)(queue);                          \
2616                      skb = tmp, tmp = skb->next)
2617 
2618 #define skb_queue_reverse_walk(queue, skb) \
2619                 for (skb = (queue)->prev;                                       \
2620                      skb != (struct sk_buff *)(queue);                          \
2621                      skb = skb->prev)
2622 
2623 #define skb_queue_reverse_walk_safe(queue, skb, tmp)                            \
2624                 for (skb = (queue)->prev, tmp = skb->prev;                      \
2625                      skb != (struct sk_buff *)(queue);                          \
2626                      skb = tmp, tmp = skb->prev)
2627 
2628 #define skb_queue_reverse_walk_from_safe(queue, skb, tmp)                       \
2629                 for (tmp = skb->prev;                                           \
2630                      skb != (struct sk_buff *)(queue);                          \
2631                      skb = tmp, tmp = skb->prev)
2632 
2633 static inline bool skb_has_frag_list(const struct sk_buff *skb)
2634 {
2635         return skb_shinfo(skb)->frag_list != NULL;
2636 }
2637 
2638 static inline void skb_frag_list_init(struct sk_buff *skb)
2639 {
2640         skb_shinfo(skb)->frag_list = NULL;
2641 }
2642 
2643 static inline void skb_frag_add_head(struct sk_buff *skb, struct sk_buff *frag)
2644 {
2645         frag->next = skb_shinfo(skb)->frag_list;
2646         skb_shinfo(skb)->frag_list = frag;
2647 }
2648 
2649 #define skb_walk_frags(skb, iter)       \
2650         for (iter = skb_shinfo(skb)->frag_list; iter; iter = iter->next)
2651 
2652 struct sk_buff *__skb_recv_datagram(struct sock *sk, unsigned flags,
2653                                     int *peeked, int *off, int *err);
2654 struct sk_buff *skb_recv_datagram(struct sock *sk, unsigned flags, int noblock,
2655                                   int *err);
2656 unsigned int datagram_poll(struct file *file, struct socket *sock,
2657                            struct poll_table_struct *wait);
2658 int skb_copy_datagram_iter(const struct sk_buff *from, int offset,
2659                            struct iov_iter *to, int size);
2660 static inline int skb_copy_datagram_msg(const struct sk_buff *from, int offset,
2661                                         struct msghdr *msg, int size)
2662 {
2663         return skb_copy_datagram_iter(from, offset, &msg->msg_iter, size);
2664 }
2665 int skb_copy_and_csum_datagram_msg(struct sk_buff *skb, int hlen,
2666                                    struct msghdr *msg);
2667 int skb_copy_datagram_from_iter(struct sk_buff *skb, int offset,
2668                                  struct iov_iter *from, int len);
2669 int zerocopy_sg_from_iter(struct sk_buff *skb, struct iov_iter *frm);
2670 void skb_free_datagram(struct sock *sk, struct sk_buff *skb);
2671 void skb_free_datagram_locked(struct sock *sk, struct sk_buff *skb);
2672 int skb_kill_datagram(struct sock *sk, struct sk_buff *skb, unsigned int flags);
2673 int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len);
2674 int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len);
2675 __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset, u8 *to,
2676                               int len, __wsum csum);
2677 int skb_splice_bits(struct sk_buff *skb, unsigned int offset,
2678                     struct pipe_inode_info *pipe, unsigned int len,
2679                     unsigned int flags);
2680 void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to);
2681 unsigned int skb_zerocopy_headlen(const struct sk_buff *from);
2682 int skb_zerocopy(struct sk_buff *to, struct sk_buff *from,
2683                  int len, int hlen);
2684 void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len);
2685 int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen);
2686 void skb_scrub_packet(struct sk_buff *skb, bool xnet);
2687 unsigned int skb_gso_transport_seglen(const struct sk_buff *skb);
2688 struct sk_buff *skb_segment(struct sk_buff *skb, netdev_features_t features);
2689 struct sk_buff *skb_vlan_untag(struct sk_buff *skb);
2690 int skb_ensure_writable(struct sk_buff *skb, int write_len);
2691 int skb_vlan_pop(struct sk_buff *skb);
2692 int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci);
2693 
2694 static inline int memcpy_from_msg(void *data, struct msghdr *msg, int len)
2695 {
2696         /* XXX: stripping const */
2697         return memcpy_fromiovec(data, (struct iovec *)msg->msg_iter.iov, len);
2698 }
2699 
2700 static inline int memcpy_to_msg(struct msghdr *msg, void *data, int len)
2701 {
2702         return copy_to_iter(data, len, &msg->msg_iter) == len ? 0 : -EFAULT;
2703 }
2704 
2705 struct skb_checksum_ops {
2706         __wsum (*update)(const void *mem, int len, __wsum wsum);
2707         __wsum (*combine)(__wsum csum, __wsum csum2, int offset, int len);
2708 };
2709 
2710 __wsum __skb_checksum(const struct sk_buff *skb, int offset, int len,
2711                       __wsum csum, const struct skb_checksum_ops *ops);
2712 __wsum skb_checksum(const struct sk_buff *skb, int offset, int len,
2713                     __wsum csum);
2714 
2715 static inline void *__skb_header_pointer(const struct sk_buff *skb, int offset,
2716                                          int len, void *data, int hlen, void *buffer)
2717 {
2718         if (hlen - offset >= len)
2719                 return data + offset;
2720 
2721         if (!skb ||
2722             skb_copy_bits(skb, offset, buffer, len) < 0)
2723                 return NULL;
2724 
2725         return buffer;
2726 }
2727 
2728 static inline void *skb_header_pointer(const struct sk_buff *skb, int offset,
2729                                        int len, void *buffer)
2730 {
2731         return __skb_header_pointer(skb, offset, len, skb->data,
2732                                     skb_headlen(skb), buffer);
2733 }
2734 
2735 /**
2736  *      skb_needs_linearize - check if we need to linearize a given skb
2737  *                            depending on the given device features.
2738  *      @skb: socket buffer to check
2739  *      @features: net device features
2740  *
2741  *      Returns true if either:
2742  *      1. skb has frag_list and the device doesn't support FRAGLIST, or
2743  *      2. skb is fragmented and the device does not support SG.
2744  */
2745 static inline bool skb_needs_linearize(struct sk_buff *skb,
2746                                        netdev_features_t features)
2747 {
2748         return skb_is_nonlinear(skb) &&
2749                ((skb_has_frag_list(skb) && !(features & NETIF_F_FRAGLIST)) ||
2750                 (skb_shinfo(skb)->nr_frags && !(features & NETIF_F_SG)));
2751 }
2752 
2753 static inline void skb_copy_from_linear_data(const struct sk_buff *skb,
2754                                              void *to,
2755                                              const unsigned int len)
2756 {
2757         memcpy(to, skb->data, len);
2758 }
2759 
2760 static inline void skb_copy_from_linear_data_offset(const struct sk_buff *skb,
2761                                                     const int offset, void *to,
2762                                                     const unsigned int len)
2763 {
2764         memcpy(to, skb->data + offset, len);
2765 }
2766 
2767 static inline void skb_copy_to_linear_data(struct sk_buff *skb,
2768                                            const void *from,
2769                                            const unsigned int len)
2770 {
2771         memcpy(skb->data, from, len);
2772 }
2773 
2774 static inline void skb_copy_to_linear_data_offset(struct sk_buff *skb,
2775                                                   const int offset,
2776                                                   const void *from,
2777                                                   const unsigned int len)
2778 {
2779         memcpy(skb->data + offset, from, len);
2780 }
2781 
2782 void skb_init(void);
2783 
2784 static inline ktime_t skb_get_ktime(const struct sk_buff *skb)
2785 {
2786         return skb->tstamp;
2787 }
2788 
2789 /**
2790  *      skb_get_timestamp - get timestamp from a skb
2791  *      @skb: skb to get stamp from
2792  *      @stamp: pointer to struct timeval to store stamp in
2793  *
2794  *      Timestamps are stored in the skb as offsets to a base timestamp.
2795  *      This function converts the offset back to a struct timeval and stores
2796  *      it in stamp.
2797  */
2798 static inline void skb_get_timestamp(const struct sk_buff *skb,
2799                                      struct timeval *stamp)
2800 {
2801         *stamp = ktime_to_timeval(skb->tstamp);
2802 }
2803 
2804 static inline void skb_get_timestampns(const struct sk_buff *skb,
2805                                        struct timespec *stamp)
2806 {
2807         *stamp = ktime_to_timespec(skb->tstamp);
2808 }
2809 
2810 static inline void __net_timestamp(struct sk_buff *skb)
2811 {
2812         skb->tstamp = ktime_get_real();
2813 }
2814 
2815 static inline ktime_t net_timedelta(ktime_t t)
2816 {
2817         return ktime_sub(ktime_get_real(), t);
2818 }
2819 
2820 static inline ktime_t net_invalid_timestamp(void)
2821 {
2822         return ktime_set(0, 0);
2823 }
2824 
2825 struct sk_buff *skb_clone_sk(struct sk_buff *skb);
2826 
2827 #ifdef CONFIG_NETWORK_PHY_TIMESTAMPING
2828 
2829 void skb_clone_tx_timestamp(struct sk_buff *skb);
2830 bool skb_defer_rx_timestamp(struct sk_buff *skb);
2831 
2832 #else /* CONFIG_NETWORK_PHY_TIMESTAMPING */
2833 
2834 static inline void skb_clone_tx_timestamp(struct sk_buff *skb)
2835 {
2836 }
2837 
2838 static inline bool skb_defer_rx_timestamp(struct sk_buff *skb)
2839 {
2840         return false;
2841 }
2842 
2843 #endif /* !CONFIG_NETWORK_PHY_TIMESTAMPING */
2844 
2845 /**
2846  * skb_complete_tx_timestamp() - deliver cloned skb with tx timestamps
2847  *
2848  * PHY drivers may accept clones of transmitted packets for
2849  * timestamping via their phy_driver.txtstamp method. These drivers
2850  * must call this function to return the skb back to the stack, with
2851  * or without a timestamp.
2852  *
2853  * @skb: clone of the the original outgoing packet
2854  * @hwtstamps: hardware time stamps, may be NULL if not available
2855  *
2856  */
2857 void skb_complete_tx_timestamp(struct sk_buff *skb,
2858                                struct skb_shared_hwtstamps *hwtstamps);
2859 
2860 void __skb_tstamp_tx(struct sk_buff *orig_skb,
2861                      struct skb_shared_hwtstamps *hwtstamps,
2862                      struct sock *sk, int tstype);
2863 
2864 /**
2865  * skb_tstamp_tx - queue clone of skb with send time stamps
2866  * @orig_skb:   the original outgoing packet
2867  * @hwtstamps:  hardware time stamps, may be NULL if not available
2868  *
2869  * If the skb has a socket associated, then this function clones the
2870  * skb (thus sharing the actual data and optional structures), stores
2871  * the optional hardware time stamping information (if non NULL) or
2872  * generates a software time stamp (otherwise), then queues the clone
2873  * to the error queue of the socket.  Errors are silently ignored.
2874  */
2875 void skb_tstamp_tx(struct sk_buff *orig_skb,
2876                    struct skb_shared_hwtstamps *hwtstamps);
2877 
2878 static inline void sw_tx_timestamp(struct sk_buff *skb)
2879 {
2880         if (skb_shinfo(skb)->tx_flags & SKBTX_SW_TSTAMP &&
2881             !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS))
2882                 skb_tstamp_tx(skb, NULL);
2883 }
2884 
2885 /**
2886  * skb_tx_timestamp() - Driver hook for transmit timestamping
2887  *
2888  * Ethernet MAC Drivers should call this function in their hard_xmit()
2889  * function immediately before giving the sk_buff to the MAC hardware.
2890  *
2891  * Specifically, one should make absolutely sure that this function is
2892  * called before TX completion of this packet can trigger.  Otherwise
2893  * the packet could potentially already be freed.
2894  *
2895  * @skb: A socket buffer.
2896  */
2897 static inline void skb_tx_timestamp(struct sk_buff *skb)
2898 {
2899         skb_clone_tx_timestamp(skb);
2900         sw_tx_timestamp(skb);
2901 }
2902 
2903 /**
2904  * skb_complete_wifi_ack - deliver skb with wifi status
2905  *
2906  * @skb: the original outgoing packet
2907  * @acked: ack status
2908  *
2909  */
2910 void skb_complete_wifi_ack(struct sk_buff *skb, bool acked);
2911 
2912 __sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len);
2913 __sum16 __skb_checksum_complete(struct sk_buff *skb);
2914 
2915 static inline int skb_csum_unnecessary(const struct sk_buff *skb)
2916 {
2917         return ((skb->ip_summed & CHECKSUM_UNNECESSARY) || skb->csum_valid);
2918 }
2919 
2920 /**
2921  *      skb_checksum_complete - Calculate checksum of an entire packet
2922  *      @skb: packet to process
2923  *
2924  *      This function calculates the checksum over the entire packet plus
2925  *      the value of skb->csum.  The latter can be used to supply the
2926  *      checksum of a pseudo header as used by TCP/UDP.  It returns the
2927  *      checksum.
2928  *
2929  *      For protocols that contain complete checksums such as ICMP/TCP/UDP,
2930  *      this function can be used to verify that checksum on received
2931  *      packets.  In that case the function should return zero if the
2932  *      checksum is correct.  In particular, this function will return zero
2933  *      if skb->ip_summed is CHECKSUM_UNNECESSARY which indicates that the
2934  *      hardware has already verified the correctness of the checksum.
2935  */
2936 static inline __sum16 skb_checksum_complete(struct sk_buff *skb)
2937 {
2938         return skb_csum_unnecessary(skb) ?
2939                0 : __skb_checksum_complete(skb);
2940 }
2941 
2942 static inline void __skb_decr_checksum_unnecessary(struct sk_buff *skb)
2943 {
2944         if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
2945                 if (skb->csum_level == 0)
2946                         skb->ip_summed = CHECKSUM_NONE;
2947                 else
2948                         skb->csum_level--;
2949         }
2950 }
2951 
2952 static inline void __skb_incr_checksum_unnecessary(struct sk_buff *skb)
2953 {
2954         if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
2955                 if (skb->csum_level < SKB_MAX_CSUM_LEVEL)
2956                         skb->csum_level++;
2957         } else if (skb->ip_summed == CHECKSUM_NONE) {
2958                 skb->ip_summed = CHECKSUM_UNNECESSARY;
2959                 skb->csum_level = 0;
2960         }
2961 }
2962 
2963 static inline void __skb_mark_checksum_bad(struct sk_buff *skb)
2964 {
2965         /* Mark current checksum as bad (typically called from GRO
2966          * path). In the case that ip_summed is CHECKSUM_NONE
2967          * this must be the first checksum encountered in the packet.
2968          * When ip_summed is CHECKSUM_UNNECESSARY, this is the first
2969          * checksum after the last one validated. For UDP, a zero
2970          * checksum can not be marked as bad.
2971          */
2972 
2973         if (skb->ip_summed == CHECKSUM_NONE ||
2974             skb->ip_summed == CHECKSUM_UNNECESSARY)
2975                 skb->csum_bad = 1;
2976 }
2977 
2978 /* Check if we need to perform checksum complete validation.
2979  *
2980  * Returns true if checksum complete is needed, false otherwise
2981  * (either checksum is unnecessary or zero checksum is allowed).
2982  */
2983 static inline bool __skb_checksum_validate_needed(struct sk_buff *skb,
2984                                                   bool zero_okay,
2985                                                   __sum16 check)
2986 {
2987         if (skb_csum_unnecessary(skb) || (zero_okay && !check)) {
2988                 skb->csum_valid = 1;
2989                 __skb_decr_checksum_unnecessary(skb);
2990                 return false;
2991         }
2992 
2993         return true;
2994 }
2995 
2996 /* For small packets <= CHECKSUM_BREAK peform checksum complete directly
2997  * in checksum_init.
2998  */
2999 #define CHECKSUM_BREAK 76
3000 
3001 /* Validate (init) checksum based on checksum complete.
3002  *
3003  * Return values:
3004  *   0: checksum is validated or try to in skb_checksum_complete. In the latter
3005  *      case the ip_summed will not be CHECKSUM_UNNECESSARY and the pseudo
3006  *      checksum is stored in skb->csum for use in __skb_checksum_complete
3007  *   non-zero: value of invalid checksum
3008  *
3009  */
3010 static inline __sum16 __skb_checksum_validate_complete(struct sk_buff *skb,
3011                                                        bool complete,
3012                                                        __wsum psum)
3013 {
3014         if (skb->ip_summed == CHECKSUM_COMPLETE) {
3015                 if (!csum_fold(csum_add(psum, skb->csum))) {
3016                         skb->csum_valid = 1;
3017                         return 0;
3018                 }
3019         } else if (skb->csum_bad) {
3020                 /* ip_summed == CHECKSUM_NONE in this case */
3021                 return 1;
3022         }
3023 
3024         skb->csum = psum;
3025 
3026         if (complete || skb->len <= CHECKSUM_BREAK) {
3027                 __sum16 csum;
3028 
3029                 csum = __skb_checksum_complete(skb);
3030                 skb->csum_valid = !csum;
3031                 return csum;
3032         }
3033 
3034         return 0;
3035 }
3036 
3037 static inline __wsum null_compute_pseudo(struct sk_buff *skb, int proto)
3038 {
3039         return 0;
3040 }
3041 
3042 /* Perform checksum validate (init). Note that this is a macro since we only
3043  * want to calculate the pseudo header which is an input function if necessary.
3044  * First we try to validate without any computation (checksum unnecessary) and
3045  * then calculate based on checksum complete calling the function to compute
3046  * pseudo header.
3047  *
3048  * Return values:
3049  *   0: checksum is validated or try to in skb_checksum_complete
3050  *   non-zero: value of invalid checksum
3051  */
3052 #define __skb_checksum_validate(skb, proto, complete,                   \
3053                                 zero_okay, check, compute_pseudo)       \
3054 ({                                                                      \
3055         __sum16 __ret = 0;                                              \
3056         skb->csum_valid = 0;                                            \
3057         if (__skb_checksum_validate_needed(skb, zero_okay, check))      \
3058                 __ret = __skb_checksum_validate_complete(skb,           \
3059                                 complete, compute_pseudo(skb, proto));  \
3060         __ret;                                                          \
3061 })
3062 
3063 #define skb_checksum_init(skb, proto, compute_pseudo)                   \
3064         __skb_checksum_validate(skb, proto, false, false, 0, compute_pseudo)
3065 
3066 #define skb_checksum_init_zero_check(skb, proto, check, compute_pseudo) \
3067         __skb_checksum_validate(skb, proto, false, true, check, compute_pseudo)
3068 
3069 #define skb_checksum_validate(skb, proto, compute_pseudo)               \
3070         __skb_checksum_validate(skb, proto, true, false, 0, compute_pseudo)
3071 
3072 #define skb_checksum_validate_zero_check(skb, proto, check,             \
3073                                          compute_pseudo)                \
3074         __skb_checksum_validate_(skb, proto, true, true, check, compute_pseudo)
3075 
3076 #define skb_checksum_simple_validate(skb)                               \
3077         __skb_checksum_validate(skb, 0, true, false, 0, null_compute_pseudo)
3078 
3079 static inline bool __skb_checksum_convert_check(struct sk_buff *skb)
3080 {
3081         return (skb->ip_summed == CHECKSUM_NONE &&
3082                 skb->csum_valid && !skb->csum_bad);
3083 }
3084 
3085 static inline void __skb_checksum_convert(struct sk_buff *skb,
3086                                           __sum16 check, __wsum pseudo)
3087 {
3088         skb->csum = ~pseudo;
3089         skb->ip_summed = CHECKSUM_COMPLETE;
3090 }
3091 
3092 #define skb_checksum_try_convert(skb, proto, check, compute_pseudo)     \
3093 do {                                                                    \
3094         if (__skb_checksum_convert_check(skb))                          \
3095                 __skb_checksum_convert(skb, check,                      \
3096                                        compute_pseudo(skb, proto));     \
3097 } while (0)
3098 
3099 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
3100 void nf_conntrack_destroy(struct nf_conntrack *nfct);
3101 static inline void nf_conntrack_put(struct nf_conntrack *nfct)
3102 {
3103         if (nfct && atomic_dec_and_test(&nfct->use))
3104                 nf_conntrack_destroy(nfct);
3105 }
3106 static inline void nf_conntrack_get(struct nf_conntrack *nfct)
3107 {
3108         if (nfct)
3109                 atomic_inc(&nfct->use);
3110 }
3111 #endif
3112 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
3113 static inline void nf_bridge_put(struct nf_bridge_info *nf_bridge)
3114 {
3115         if (nf_bridge && atomic_dec_and_test(&nf_bridge->use))
3116                 kfree(nf_bridge);
3117 }
3118 static inline void nf_bridge_get(struct nf_bridge_info *nf_bridge)
3119 {
3120         if (nf_bridge)
3121                 atomic_inc(&nf_bridge->use);
3122 }
3123 #endif /* CONFIG_BRIDGE_NETFILTER */
3124 static inline void nf_reset(struct sk_buff *skb)
3125 {
3126 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
3127         nf_conntrack_put(skb->nfct);
3128         skb->nfct = NULL;
3129 #endif
3130 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
3131         nf_bridge_put(skb->nf_bridge);
3132         skb->nf_bridge = NULL;
3133 #endif
3134 }
3135 
3136 static inline void nf_reset_trace(struct sk_buff *skb)
3137 {
3138 #if IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TRACE) || defined(CONFIG_NF_TABLES)
3139         skb->nf_trace = 0;
3140 #endif
3141 }
3142 
3143 /* Note: This doesn't put any conntrack and bridge info in dst. */
3144 static inline void __nf_copy(struct sk_buff *dst, const struct sk_buff *src,
3145                              bool copy)
3146 {
3147 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
3148         dst->nfct = src->nfct;
3149         nf_conntrack_get(src->nfct);
3150         if (copy)
3151                 dst->nfctinfo = src->nfctinfo;
3152 #endif
3153 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
3154         dst->nf_bridge  = src->nf_bridge;
3155         nf_bridge_get(src->nf_bridge);
3156 #endif
3157 #if IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TRACE) || defined(CONFIG_NF_TABLES)
3158         if (copy)
3159                 dst->nf_trace = src->nf_trace;
3160 #endif
3161 }
3162 
3163 static inline void nf_copy(struct sk_buff *dst, const struct sk_buff *src)
3164 {
3165 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
3166         nf_conntrack_put(dst->nfct);
3167 #endif
3168 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
3169         nf_bridge_put(dst->nf_bridge);
3170 #endif
3171         __nf_copy(dst, src, true);
3172 }
3173 
3174 #ifdef CONFIG_NETWORK_SECMARK
3175 static inline void skb_copy_secmark(struct sk_buff *to, const struct sk_buff *from)
3176 {
3177         to->secmark = from->secmark;
3178 }
3179 
3180 static inline void skb_init_secmark(struct sk_buff *skb)
3181 {
3182         skb->secmark = 0;
3183 }
3184 #else
3185 static inline void skb_copy_secmark(struct sk_buff *to, const struct sk_buff *from)
3186 { }
3187 
3188 static inline void skb_init_secmark(struct sk_buff *skb)
3189 { }
3190 #endif
3191 
3192 static inline bool skb_irq_freeable(const struct sk_buff *skb)
3193 {
3194         return !skb->destructor &&
3195 #if IS_ENABLED(CONFIG_XFRM)
3196                 !skb->sp &&
3197 #endif
3198 #if IS_ENABLED(CONFIG_NF_CONNTRACK)
3199                 !skb->nfct &&
3200 #endif
3201                 !skb->_skb_refdst &&
3202                 !skb_has_frag_list(skb);
3203 }
3204 
3205 static inline void skb_set_queue_mapping(struct sk_buff *skb, u16 queue_mapping)
3206 {
3207         skb->queue_mapping = queue_mapping;
3208 }
3209 
3210 static inline u16 skb_get_queue_mapping(const struct sk_buff *skb)
3211 {
3212         return skb->queue_mapping;
3213 }
3214 
3215 static inline void skb_copy_queue_mapping(struct sk_buff *to, const struct sk_buff *from)
3216 {
3217         to->queue_mapping = from->queue_mapping;
3218 }
3219 
3220 static inline void skb_record_rx_queue(struct sk_buff *skb, u16 rx_queue)
3221 {
3222         skb->queue_mapping = rx_queue + 1;
3223 }
3224 
3225 static inline u16 skb_get_rx_queue(const struct sk_buff *skb)
3226 {
3227         return skb->queue_mapping - 1;
3228 }
3229 
3230 static inline bool skb_rx_queue_recorded(const struct sk_buff *skb)
3231 {
3232         return skb->queue_mapping != 0;
3233 }
3234 
3235 u16 __skb_tx_hash(const struct net_device *dev, struct sk_buff *skb,
3236                   unsigned int num_tx_queues);
3237 
3238 static inline struct sec_path *skb_sec_path(struct sk_buff *skb)
3239 {
3240 #ifdef CONFIG_XFRM
3241         return skb->sp;
3242 #else
3243         return NULL;
3244 #endif
3245 }
3246 
3247 /* Keeps track of mac header offset relative to skb->head.
3248  * It is useful for TSO of Tunneling protocol. e.g. GRE.
3249  * For non-tunnel skb it points to skb_mac_header() and for
3250  * tunnel skb it points to outer mac header.
3251  * Keeps track of level of encapsulation of network headers.
3252  */
3253 struct skb_gso_cb {
3254         int     mac_offset;
3255         int     encap_level;
3256         __u16   csum_start;
3257 };
3258 #define SKB_GSO_CB(skb) ((struct skb_gso_cb *)(skb)->cb)
3259 
3260 static inline int skb_tnl_header_len(const struct sk_buff *inner_skb)
3261 {
3262         return (skb_mac_header(inner_skb) - inner_skb->head) -
3263                 SKB_GSO_CB(inner_skb)->mac_offset;
3264 }
3265 
3266 static inline int gso_pskb_expand_head(struct sk_buff *skb, int extra)
3267 {
3268         int new_headroom, headroom;
3269         int ret;
3270 
3271         headroom = skb_headroom(skb);
3272         ret = pskb_expand_head(skb, extra, 0, GFP_ATOMIC);
3273         if (ret)
3274                 return ret;
3275 
3276         new_headroom = skb_headroom(skb);
3277         SKB_GSO_CB(skb)->mac_offset += (new_headroom - headroom);
3278         return 0;
3279 }
3280 
3281 /* Compute the checksum for a gso segment. First compute the checksum value
3282  * from the start of transport header to SKB_GSO_CB(skb)->csum_start, and
3283  * then add in skb->csum (checksum from csum_start to end of packet).
3284  * skb->csum and csum_start are then updated to reflect the checksum of the
3285  * resultant packet starting from the transport header-- the resultant checksum
3286  * is in the res argument (i.e. normally zero or ~ of checksum of a pseudo
3287  * header.
3288  */
3289 static inline __sum16 gso_make_checksum(struct sk_buff *skb, __wsum res)
3290 {
3291         int plen = SKB_GSO_CB(skb)->csum_start - skb_headroom(skb) -
3292             skb_transport_offset(skb);
3293         __u16 csum;
3294 
3295         csum = csum_fold(csum_partial(skb_transport_header(skb),
3296                                       plen, skb->csum));
3297         skb->csum = res;
3298         SKB_GSO_CB(skb)->csum_start -= plen;
3299 
3300         return csum;
3301 }
3302 
3303 static inline bool skb_is_gso(const struct sk_buff *skb)
3304 {
3305         return skb_shinfo(skb)->gso_size;
3306 }
3307 
3308 /* Note: Should be called only if skb_is_gso(skb) is true */
3309 static inline bool skb_is_gso_v6(const struct sk_buff *skb)
3310 {
3311         return skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6;
3312 }
3313 
3314 void __skb_warn_lro_forwarding(const struct sk_buff *skb);
3315 
3316 static inline bool skb_warn_if_lro(const struct sk_buff *skb)
3317 {
3318         /* LRO sets gso_size but not gso_type, whereas if GSO is really
3319          * wanted then gso_type will be set. */
3320         const struct skb_shared_info *shinfo = skb_shinfo(skb);
3321 
3322         if (skb_is_nonlinear(skb) && shinfo->gso_size != 0 &&
3323             unlikely(shinfo->gso_type == 0)) {
3324                 __skb_warn_lro_forwarding(skb);
3325                 return true;
3326         }
3327         return false;
3328 }
3329 
3330 static inline void skb_forward_csum(struct sk_buff *skb)
3331 {
3332         /* Unfortunately we don't support this one.  Any brave souls? */
3333         if (skb->ip_summed == CHECKSUM_COMPLETE)
3334                 skb->ip_summed = CHECKSUM_NONE;
3335 }
3336 
3337 /**
3338  * skb_checksum_none_assert - make sure skb ip_summed is CHECKSUM_NONE
3339  * @skb: skb to check
3340  *
3341  * fresh skbs have their ip_summed set to CHECKSUM_NONE.
3342  * Instead of forcing ip_summed to CHECKSUM_NONE, we can
3343  * use this helper, to document places where we make this assertion.
3344  */
3345 static inline void skb_checksum_none_assert(const struct sk_buff *skb)
3346 {
3347 #ifdef DEBUG
3348         BUG_ON(skb->ip_summed != CHECKSUM_NONE);
3349 #endif
3350 }
3351 
3352 bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off);
3353 
3354 int skb_checksum_setup(struct sk_buff *skb, bool recalculate);
3355 
3356 u32 skb_get_poff(const struct sk_buff *skb);
3357 u32 __skb_get_poff(const struct sk_buff *skb, void *data,
3358                    const struct flow_keys *keys, int hlen);
3359 
3360 /**
3361  * skb_head_is_locked - Determine if the skb->head is locked down
3362  * @skb: skb to check
3363  *
3364  * The head on skbs build around a head frag can be removed if they are
3365  * not cloned.  This function returns true if the skb head is locked down
3366  * due to either being allocated via kmalloc, or by being a clone with
3367  * multiple references to the head.
3368  */
3369 static inline bool skb_head_is_locked(const struct sk_buff *skb)
3370 {
3371         return !skb->head_frag || skb_cloned(skb);
3372 }
3373 
3374 /**
3375  * skb_gso_network_seglen - Return length of individual segments of a gso packet
3376  *
3377  * @skb: GSO skb
3378  *
3379  * skb_gso_network_seglen is used to determine the real size of the
3380  * individual segments, including Layer3 (IP, IPv6) and L4 headers (TCP/UDP).
3381  *
3382  * The MAC/L2 header is not accounted for.
3383  */
3384 static inline unsigned int skb_gso_network_seglen(const struct sk_buff *skb)
3385 {
3386         unsigned int hdr_len = skb_transport_header(skb) -
3387                                skb_network_header(skb);
3388         return hdr_len + skb_gso_transport_seglen(skb);
3389 }
3390 #endif  /* __KERNEL__ */
3391 #endif  /* _LINUX_SKBUFF_H */
3392 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us