Version:  2.0.40 2.2.26 2.4.37 3.8 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1 4.2 4.3 4.4 4.5

Linux/net/core/dev.c

  1 /*
  2  *      NET3    Protocol independent device support routines.
  3  *
  4  *              This program is free software; you can redistribute it and/or
  5  *              modify it under the terms of the GNU General Public License
  6  *              as published by the Free Software Foundation; either version
  7  *              2 of the License, or (at your option) any later version.
  8  *
  9  *      Derived from the non IP parts of dev.c 1.0.19
 10  *              Authors:        Ross Biro
 11  *                              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
 12  *                              Mark Evans, <evansmp@uhura.aston.ac.uk>
 13  *
 14  *      Additional Authors:
 15  *              Florian la Roche <rzsfl@rz.uni-sb.de>
 16  *              Alan Cox <gw4pts@gw4pts.ampr.org>
 17  *              David Hinds <dahinds@users.sourceforge.net>
 18  *              Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
 19  *              Adam Sulmicki <adam@cfar.umd.edu>
 20  *              Pekka Riikonen <priikone@poesidon.pspt.fi>
 21  *
 22  *      Changes:
 23  *              D.J. Barrow     :       Fixed bug where dev->refcnt gets set
 24  *                                      to 2 if register_netdev gets called
 25  *                                      before net_dev_init & also removed a
 26  *                                      few lines of code in the process.
 27  *              Alan Cox        :       device private ioctl copies fields back.
 28  *              Alan Cox        :       Transmit queue code does relevant
 29  *                                      stunts to keep the queue safe.
 30  *              Alan Cox        :       Fixed double lock.
 31  *              Alan Cox        :       Fixed promisc NULL pointer trap
 32  *              ????????        :       Support the full private ioctl range
 33  *              Alan Cox        :       Moved ioctl permission check into
 34  *                                      drivers
 35  *              Tim Kordas      :       SIOCADDMULTI/SIOCDELMULTI
 36  *              Alan Cox        :       100 backlog just doesn't cut it when
 37  *                                      you start doing multicast video 8)
 38  *              Alan Cox        :       Rewrote net_bh and list manager.
 39  *              Alan Cox        :       Fix ETH_P_ALL echoback lengths.
 40  *              Alan Cox        :       Took out transmit every packet pass
 41  *                                      Saved a few bytes in the ioctl handler
 42  *              Alan Cox        :       Network driver sets packet type before
 43  *                                      calling netif_rx. Saves a function
 44  *                                      call a packet.
 45  *              Alan Cox        :       Hashed net_bh()
 46  *              Richard Kooijman:       Timestamp fixes.
 47  *              Alan Cox        :       Wrong field in SIOCGIFDSTADDR
 48  *              Alan Cox        :       Device lock protection.
 49  *              Alan Cox        :       Fixed nasty side effect of device close
 50  *                                      changes.
 51  *              Rudi Cilibrasi  :       Pass the right thing to
 52  *                                      set_mac_address()
 53  *              Dave Miller     :       32bit quantity for the device lock to
 54  *                                      make it work out on a Sparc.
 55  *              Bjorn Ekwall    :       Added KERNELD hack.
 56  *              Alan Cox        :       Cleaned up the backlog initialise.
 57  *              Craig Metz      :       SIOCGIFCONF fix if space for under
 58  *                                      1 device.
 59  *          Thomas Bogendoerfer :       Return ENODEV for dev_open, if there
 60  *                                      is no device open function.
 61  *              Andi Kleen      :       Fix error reporting for SIOCGIFCONF
 62  *          Michael Chastain    :       Fix signed/unsigned for SIOCGIFCONF
 63  *              Cyrus Durgin    :       Cleaned for KMOD
 64  *              Adam Sulmicki   :       Bug Fix : Network Device Unload
 65  *                                      A network device unload needs to purge
 66  *                                      the backlog queue.
 67  *      Paul Rusty Russell      :       SIOCSIFNAME
 68  *              Pekka Riikonen  :       Netdev boot-time settings code
 69  *              Andrew Morton   :       Make unregister_netdevice wait
 70  *                                      indefinitely on dev->refcnt
 71  *              J Hadi Salim    :       - Backlog queue sampling
 72  *                                      - netif_rx() feedback
 73  */
 74 
 75 #include <asm/uaccess.h>
 76 #include <linux/bitops.h>
 77 #include <linux/capability.h>
 78 #include <linux/cpu.h>
 79 #include <linux/types.h>
 80 #include <linux/kernel.h>
 81 #include <linux/hash.h>
 82 #include <linux/slab.h>
 83 #include <linux/sched.h>
 84 #include <linux/mutex.h>
 85 #include <linux/string.h>
 86 #include <linux/mm.h>
 87 #include <linux/socket.h>
 88 #include <linux/sockios.h>
 89 #include <linux/errno.h>
 90 #include <linux/interrupt.h>
 91 #include <linux/if_ether.h>
 92 #include <linux/netdevice.h>
 93 #include <linux/etherdevice.h>
 94 #include <linux/ethtool.h>
 95 #include <linux/notifier.h>
 96 #include <linux/skbuff.h>
 97 #include <net/net_namespace.h>
 98 #include <net/sock.h>
 99 #include <net/busy_poll.h>
100 #include <linux/rtnetlink.h>
101 #include <linux/stat.h>
102 #include <net/dst.h>
103 #include <net/dst_metadata.h>
104 #include <net/pkt_sched.h>
105 #include <net/checksum.h>
106 #include <net/xfrm.h>
107 #include <linux/highmem.h>
108 #include <linux/init.h>
109 #include <linux/module.h>
110 #include <linux/netpoll.h>
111 #include <linux/rcupdate.h>
112 #include <linux/delay.h>
113 #include <net/iw_handler.h>
114 #include <asm/current.h>
115 #include <linux/audit.h>
116 #include <linux/dmaengine.h>
117 #include <linux/err.h>
118 #include <linux/ctype.h>
119 #include <linux/if_arp.h>
120 #include <linux/if_vlan.h>
121 #include <linux/ip.h>
122 #include <net/ip.h>
123 #include <net/mpls.h>
124 #include <linux/ipv6.h>
125 #include <linux/in.h>
126 #include <linux/jhash.h>
127 #include <linux/random.h>
128 #include <trace/events/napi.h>
129 #include <trace/events/net.h>
130 #include <trace/events/skb.h>
131 #include <linux/pci.h>
132 #include <linux/inetdevice.h>
133 #include <linux/cpu_rmap.h>
134 #include <linux/static_key.h>
135 #include <linux/hashtable.h>
136 #include <linux/vmalloc.h>
137 #include <linux/if_macvlan.h>
138 #include <linux/errqueue.h>
139 #include <linux/hrtimer.h>
140 #include <linux/netfilter_ingress.h>
141 #include <linux/sctp.h>
142 
143 #include "net-sysfs.h"
144 
145 /* Instead of increasing this, you should create a hash table. */
146 #define MAX_GRO_SKBS 8
147 
148 /* This should be increased if a protocol with a bigger head is added. */
149 #define GRO_MAX_HEAD (MAX_HEADER + 128)
150 
151 static DEFINE_SPINLOCK(ptype_lock);
152 static DEFINE_SPINLOCK(offload_lock);
153 struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
154 struct list_head ptype_all __read_mostly;       /* Taps */
155 static struct list_head offload_base __read_mostly;
156 
157 static int netif_rx_internal(struct sk_buff *skb);
158 static int call_netdevice_notifiers_info(unsigned long val,
159                                          struct net_device *dev,
160                                          struct netdev_notifier_info *info);
161 
162 /*
163  * The @dev_base_head list is protected by @dev_base_lock and the rtnl
164  * semaphore.
165  *
166  * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
167  *
168  * Writers must hold the rtnl semaphore while they loop through the
169  * dev_base_head list, and hold dev_base_lock for writing when they do the
170  * actual updates.  This allows pure readers to access the list even
171  * while a writer is preparing to update it.
172  *
173  * To put it another way, dev_base_lock is held for writing only to
174  * protect against pure readers; the rtnl semaphore provides the
175  * protection against other writers.
176  *
177  * See, for example usages, register_netdevice() and
178  * unregister_netdevice(), which must be called with the rtnl
179  * semaphore held.
180  */
181 DEFINE_RWLOCK(dev_base_lock);
182 EXPORT_SYMBOL(dev_base_lock);
183 
184 /* protects napi_hash addition/deletion and napi_gen_id */
185 static DEFINE_SPINLOCK(napi_hash_lock);
186 
187 static unsigned int napi_gen_id = NR_CPUS;
188 static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8);
189 
190 static seqcount_t devnet_rename_seq;
191 
192 static inline void dev_base_seq_inc(struct net *net)
193 {
194         while (++net->dev_base_seq == 0);
195 }
196 
197 static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
198 {
199         unsigned int hash = full_name_hash(name, strnlen(name, IFNAMSIZ));
200 
201         return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
202 }
203 
204 static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
205 {
206         return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
207 }
208 
209 static inline void rps_lock(struct softnet_data *sd)
210 {
211 #ifdef CONFIG_RPS
212         spin_lock(&sd->input_pkt_queue.lock);
213 #endif
214 }
215 
216 static inline void rps_unlock(struct softnet_data *sd)
217 {
218 #ifdef CONFIG_RPS
219         spin_unlock(&sd->input_pkt_queue.lock);
220 #endif
221 }
222 
223 /* Device list insertion */
224 static void list_netdevice(struct net_device *dev)
225 {
226         struct net *net = dev_net(dev);
227 
228         ASSERT_RTNL();
229 
230         write_lock_bh(&dev_base_lock);
231         list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
232         hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
233         hlist_add_head_rcu(&dev->index_hlist,
234                            dev_index_hash(net, dev->ifindex));
235         write_unlock_bh(&dev_base_lock);
236 
237         dev_base_seq_inc(net);
238 }
239 
240 /* Device list removal
241  * caller must respect a RCU grace period before freeing/reusing dev
242  */
243 static void unlist_netdevice(struct net_device *dev)
244 {
245         ASSERT_RTNL();
246 
247         /* Unlink dev from the device chain */
248         write_lock_bh(&dev_base_lock);
249         list_del_rcu(&dev->dev_list);
250         hlist_del_rcu(&dev->name_hlist);
251         hlist_del_rcu(&dev->index_hlist);
252         write_unlock_bh(&dev_base_lock);
253 
254         dev_base_seq_inc(dev_net(dev));
255 }
256 
257 /*
258  *      Our notifier list
259  */
260 
261 static RAW_NOTIFIER_HEAD(netdev_chain);
262 
263 /*
264  *      Device drivers call our routines to queue packets here. We empty the
265  *      queue in the local softnet handler.
266  */
267 
268 DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
269 EXPORT_PER_CPU_SYMBOL(softnet_data);
270 
271 #ifdef CONFIG_LOCKDEP
272 /*
273  * register_netdevice() inits txq->_xmit_lock and sets lockdep class
274  * according to dev->type
275  */
276 static const unsigned short netdev_lock_type[] =
277         {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
278          ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
279          ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
280          ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
281          ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
282          ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
283          ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
284          ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
285          ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
286          ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
287          ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
288          ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
289          ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
290          ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
291          ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
292 
293 static const char *const netdev_lock_name[] =
294         {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
295          "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
296          "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
297          "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
298          "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
299          "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
300          "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
301          "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
302          "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
303          "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
304          "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
305          "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
306          "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
307          "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
308          "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
309 
310 static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
311 static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
312 
313 static inline unsigned short netdev_lock_pos(unsigned short dev_type)
314 {
315         int i;
316 
317         for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
318                 if (netdev_lock_type[i] == dev_type)
319                         return i;
320         /* the last key is used by default */
321         return ARRAY_SIZE(netdev_lock_type) - 1;
322 }
323 
324 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
325                                                  unsigned short dev_type)
326 {
327         int i;
328 
329         i = netdev_lock_pos(dev_type);
330         lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
331                                    netdev_lock_name[i]);
332 }
333 
334 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
335 {
336         int i;
337 
338         i = netdev_lock_pos(dev->type);
339         lockdep_set_class_and_name(&dev->addr_list_lock,
340                                    &netdev_addr_lock_key[i],
341                                    netdev_lock_name[i]);
342 }
343 #else
344 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
345                                                  unsigned short dev_type)
346 {
347 }
348 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
349 {
350 }
351 #endif
352 
353 /*******************************************************************************
354 
355                 Protocol management and registration routines
356 
357 *******************************************************************************/
358 
359 /*
360  *      Add a protocol ID to the list. Now that the input handler is
361  *      smarter we can dispense with all the messy stuff that used to be
362  *      here.
363  *
364  *      BEWARE!!! Protocol handlers, mangling input packets,
365  *      MUST BE last in hash buckets and checking protocol handlers
366  *      MUST start from promiscuous ptype_all chain in net_bh.
367  *      It is true now, do not change it.
368  *      Explanation follows: if protocol handler, mangling packet, will
369  *      be the first on list, it is not able to sense, that packet
370  *      is cloned and should be copied-on-write, so that it will
371  *      change it and subsequent readers will get broken packet.
372  *                                                      --ANK (980803)
373  */
374 
375 static inline struct list_head *ptype_head(const struct packet_type *pt)
376 {
377         if (pt->type == htons(ETH_P_ALL))
378                 return pt->dev ? &pt->dev->ptype_all : &ptype_all;
379         else
380                 return pt->dev ? &pt->dev->ptype_specific :
381                                  &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
382 }
383 
384 /**
385  *      dev_add_pack - add packet handler
386  *      @pt: packet type declaration
387  *
388  *      Add a protocol handler to the networking stack. The passed &packet_type
389  *      is linked into kernel lists and may not be freed until it has been
390  *      removed from the kernel lists.
391  *
392  *      This call does not sleep therefore it can not
393  *      guarantee all CPU's that are in middle of receiving packets
394  *      will see the new packet type (until the next received packet).
395  */
396 
397 void dev_add_pack(struct packet_type *pt)
398 {
399         struct list_head *head = ptype_head(pt);
400 
401         spin_lock(&ptype_lock);
402         list_add_rcu(&pt->list, head);
403         spin_unlock(&ptype_lock);
404 }
405 EXPORT_SYMBOL(dev_add_pack);
406 
407 /**
408  *      __dev_remove_pack        - remove packet handler
409  *      @pt: packet type declaration
410  *
411  *      Remove a protocol handler that was previously added to the kernel
412  *      protocol handlers by dev_add_pack(). The passed &packet_type is removed
413  *      from the kernel lists and can be freed or reused once this function
414  *      returns.
415  *
416  *      The packet type might still be in use by receivers
417  *      and must not be freed until after all the CPU's have gone
418  *      through a quiescent state.
419  */
420 void __dev_remove_pack(struct packet_type *pt)
421 {
422         struct list_head *head = ptype_head(pt);
423         struct packet_type *pt1;
424 
425         spin_lock(&ptype_lock);
426 
427         list_for_each_entry(pt1, head, list) {
428                 if (pt == pt1) {
429                         list_del_rcu(&pt->list);
430                         goto out;
431                 }
432         }
433 
434         pr_warn("dev_remove_pack: %p not found\n", pt);
435 out:
436         spin_unlock(&ptype_lock);
437 }
438 EXPORT_SYMBOL(__dev_remove_pack);
439 
440 /**
441  *      dev_remove_pack  - remove packet handler
442  *      @pt: packet type declaration
443  *
444  *      Remove a protocol handler that was previously added to the kernel
445  *      protocol handlers by dev_add_pack(). The passed &packet_type is removed
446  *      from the kernel lists and can be freed or reused once this function
447  *      returns.
448  *
449  *      This call sleeps to guarantee that no CPU is looking at the packet
450  *      type after return.
451  */
452 void dev_remove_pack(struct packet_type *pt)
453 {
454         __dev_remove_pack(pt);
455 
456         synchronize_net();
457 }
458 EXPORT_SYMBOL(dev_remove_pack);
459 
460 
461 /**
462  *      dev_add_offload - register offload handlers
463  *      @po: protocol offload declaration
464  *
465  *      Add protocol offload handlers to the networking stack. The passed
466  *      &proto_offload is linked into kernel lists and may not be freed until
467  *      it has been removed from the kernel lists.
468  *
469  *      This call does not sleep therefore it can not
470  *      guarantee all CPU's that are in middle of receiving packets
471  *      will see the new offload handlers (until the next received packet).
472  */
473 void dev_add_offload(struct packet_offload *po)
474 {
475         struct packet_offload *elem;
476 
477         spin_lock(&offload_lock);
478         list_for_each_entry(elem, &offload_base, list) {
479                 if (po->priority < elem->priority)
480                         break;
481         }
482         list_add_rcu(&po->list, elem->list.prev);
483         spin_unlock(&offload_lock);
484 }
485 EXPORT_SYMBOL(dev_add_offload);
486 
487 /**
488  *      __dev_remove_offload     - remove offload handler
489  *      @po: packet offload declaration
490  *
491  *      Remove a protocol offload handler that was previously added to the
492  *      kernel offload handlers by dev_add_offload(). The passed &offload_type
493  *      is removed from the kernel lists and can be freed or reused once this
494  *      function returns.
495  *
496  *      The packet type might still be in use by receivers
497  *      and must not be freed until after all the CPU's have gone
498  *      through a quiescent state.
499  */
500 static void __dev_remove_offload(struct packet_offload *po)
501 {
502         struct list_head *head = &offload_base;
503         struct packet_offload *po1;
504 
505         spin_lock(&offload_lock);
506 
507         list_for_each_entry(po1, head, list) {
508                 if (po == po1) {
509                         list_del_rcu(&po->list);
510                         goto out;
511                 }
512         }
513 
514         pr_warn("dev_remove_offload: %p not found\n", po);
515 out:
516         spin_unlock(&offload_lock);
517 }
518 
519 /**
520  *      dev_remove_offload       - remove packet offload handler
521  *      @po: packet offload declaration
522  *
523  *      Remove a packet offload handler that was previously added to the kernel
524  *      offload handlers by dev_add_offload(). The passed &offload_type is
525  *      removed from the kernel lists and can be freed or reused once this
526  *      function returns.
527  *
528  *      This call sleeps to guarantee that no CPU is looking at the packet
529  *      type after return.
530  */
531 void dev_remove_offload(struct packet_offload *po)
532 {
533         __dev_remove_offload(po);
534 
535         synchronize_net();
536 }
537 EXPORT_SYMBOL(dev_remove_offload);
538 
539 /******************************************************************************
540 
541                       Device Boot-time Settings Routines
542 
543 *******************************************************************************/
544 
545 /* Boot time configuration table */
546 static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
547 
548 /**
549  *      netdev_boot_setup_add   - add new setup entry
550  *      @name: name of the device
551  *      @map: configured settings for the device
552  *
553  *      Adds new setup entry to the dev_boot_setup list.  The function
554  *      returns 0 on error and 1 on success.  This is a generic routine to
555  *      all netdevices.
556  */
557 static int netdev_boot_setup_add(char *name, struct ifmap *map)
558 {
559         struct netdev_boot_setup *s;
560         int i;
561 
562         s = dev_boot_setup;
563         for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
564                 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
565                         memset(s[i].name, 0, sizeof(s[i].name));
566                         strlcpy(s[i].name, name, IFNAMSIZ);
567                         memcpy(&s[i].map, map, sizeof(s[i].map));
568                         break;
569                 }
570         }
571 
572         return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
573 }
574 
575 /**
576  *      netdev_boot_setup_check - check boot time settings
577  *      @dev: the netdevice
578  *
579  *      Check boot time settings for the device.
580  *      The found settings are set for the device to be used
581  *      later in the device probing.
582  *      Returns 0 if no settings found, 1 if they are.
583  */
584 int netdev_boot_setup_check(struct net_device *dev)
585 {
586         struct netdev_boot_setup *s = dev_boot_setup;
587         int i;
588 
589         for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
590                 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
591                     !strcmp(dev->name, s[i].name)) {
592                         dev->irq        = s[i].map.irq;
593                         dev->base_addr  = s[i].map.base_addr;
594                         dev->mem_start  = s[i].map.mem_start;
595                         dev->mem_end    = s[i].map.mem_end;
596                         return 1;
597                 }
598         }
599         return 0;
600 }
601 EXPORT_SYMBOL(netdev_boot_setup_check);
602 
603 
604 /**
605  *      netdev_boot_base        - get address from boot time settings
606  *      @prefix: prefix for network device
607  *      @unit: id for network device
608  *
609  *      Check boot time settings for the base address of device.
610  *      The found settings are set for the device to be used
611  *      later in the device probing.
612  *      Returns 0 if no settings found.
613  */
614 unsigned long netdev_boot_base(const char *prefix, int unit)
615 {
616         const struct netdev_boot_setup *s = dev_boot_setup;
617         char name[IFNAMSIZ];
618         int i;
619 
620         sprintf(name, "%s%d", prefix, unit);
621 
622         /*
623          * If device already registered then return base of 1
624          * to indicate not to probe for this interface
625          */
626         if (__dev_get_by_name(&init_net, name))
627                 return 1;
628 
629         for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
630                 if (!strcmp(name, s[i].name))
631                         return s[i].map.base_addr;
632         return 0;
633 }
634 
635 /*
636  * Saves at boot time configured settings for any netdevice.
637  */
638 int __init netdev_boot_setup(char *str)
639 {
640         int ints[5];
641         struct ifmap map;
642 
643         str = get_options(str, ARRAY_SIZE(ints), ints);
644         if (!str || !*str)
645                 return 0;
646 
647         /* Save settings */
648         memset(&map, 0, sizeof(map));
649         if (ints[0] > 0)
650                 map.irq = ints[1];
651         if (ints[0] > 1)
652                 map.base_addr = ints[2];
653         if (ints[0] > 2)
654                 map.mem_start = ints[3];
655         if (ints[0] > 3)
656                 map.mem_end = ints[4];
657 
658         /* Add new entry to the list */
659         return netdev_boot_setup_add(str, &map);
660 }
661 
662 __setup("netdev=", netdev_boot_setup);
663 
664 /*******************************************************************************
665 
666                             Device Interface Subroutines
667 
668 *******************************************************************************/
669 
670 /**
671  *      dev_get_iflink  - get 'iflink' value of a interface
672  *      @dev: targeted interface
673  *
674  *      Indicates the ifindex the interface is linked to.
675  *      Physical interfaces have the same 'ifindex' and 'iflink' values.
676  */
677 
678 int dev_get_iflink(const struct net_device *dev)
679 {
680         if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
681                 return dev->netdev_ops->ndo_get_iflink(dev);
682 
683         return dev->ifindex;
684 }
685 EXPORT_SYMBOL(dev_get_iflink);
686 
687 /**
688  *      dev_fill_metadata_dst - Retrieve tunnel egress information.
689  *      @dev: targeted interface
690  *      @skb: The packet.
691  *
692  *      For better visibility of tunnel traffic OVS needs to retrieve
693  *      egress tunnel information for a packet. Following API allows
694  *      user to get this info.
695  */
696 int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
697 {
698         struct ip_tunnel_info *info;
699 
700         if (!dev->netdev_ops  || !dev->netdev_ops->ndo_fill_metadata_dst)
701                 return -EINVAL;
702 
703         info = skb_tunnel_info_unclone(skb);
704         if (!info)
705                 return -ENOMEM;
706         if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
707                 return -EINVAL;
708 
709         return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
710 }
711 EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
712 
713 /**
714  *      __dev_get_by_name       - find a device by its name
715  *      @net: the applicable net namespace
716  *      @name: name to find
717  *
718  *      Find an interface by name. Must be called under RTNL semaphore
719  *      or @dev_base_lock. If the name is found a pointer to the device
720  *      is returned. If the name is not found then %NULL is returned. The
721  *      reference counters are not incremented so the caller must be
722  *      careful with locks.
723  */
724 
725 struct net_device *__dev_get_by_name(struct net *net, const char *name)
726 {
727         struct net_device *dev;
728         struct hlist_head *head = dev_name_hash(net, name);
729 
730         hlist_for_each_entry(dev, head, name_hlist)
731                 if (!strncmp(dev->name, name, IFNAMSIZ))
732                         return dev;
733 
734         return NULL;
735 }
736 EXPORT_SYMBOL(__dev_get_by_name);
737 
738 /**
739  *      dev_get_by_name_rcu     - find a device by its name
740  *      @net: the applicable net namespace
741  *      @name: name to find
742  *
743  *      Find an interface by name.
744  *      If the name is found a pointer to the device is returned.
745  *      If the name is not found then %NULL is returned.
746  *      The reference counters are not incremented so the caller must be
747  *      careful with locks. The caller must hold RCU lock.
748  */
749 
750 struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
751 {
752         struct net_device *dev;
753         struct hlist_head *head = dev_name_hash(net, name);
754 
755         hlist_for_each_entry_rcu(dev, head, name_hlist)
756                 if (!strncmp(dev->name, name, IFNAMSIZ))
757                         return dev;
758 
759         return NULL;
760 }
761 EXPORT_SYMBOL(dev_get_by_name_rcu);
762 
763 /**
764  *      dev_get_by_name         - find a device by its name
765  *      @net: the applicable net namespace
766  *      @name: name to find
767  *
768  *      Find an interface by name. This can be called from any
769  *      context and does its own locking. The returned handle has
770  *      the usage count incremented and the caller must use dev_put() to
771  *      release it when it is no longer needed. %NULL is returned if no
772  *      matching device is found.
773  */
774 
775 struct net_device *dev_get_by_name(struct net *net, const char *name)
776 {
777         struct net_device *dev;
778 
779         rcu_read_lock();
780         dev = dev_get_by_name_rcu(net, name);
781         if (dev)
782                 dev_hold(dev);
783         rcu_read_unlock();
784         return dev;
785 }
786 EXPORT_SYMBOL(dev_get_by_name);
787 
788 /**
789  *      __dev_get_by_index - find a device by its ifindex
790  *      @net: the applicable net namespace
791  *      @ifindex: index of device
792  *
793  *      Search for an interface by index. Returns %NULL if the device
794  *      is not found or a pointer to the device. The device has not
795  *      had its reference counter increased so the caller must be careful
796  *      about locking. The caller must hold either the RTNL semaphore
797  *      or @dev_base_lock.
798  */
799 
800 struct net_device *__dev_get_by_index(struct net *net, int ifindex)
801 {
802         struct net_device *dev;
803         struct hlist_head *head = dev_index_hash(net, ifindex);
804 
805         hlist_for_each_entry(dev, head, index_hlist)
806                 if (dev->ifindex == ifindex)
807                         return dev;
808 
809         return NULL;
810 }
811 EXPORT_SYMBOL(__dev_get_by_index);
812 
813 /**
814  *      dev_get_by_index_rcu - find a device by its ifindex
815  *      @net: the applicable net namespace
816  *      @ifindex: index of device
817  *
818  *      Search for an interface by index. Returns %NULL if the device
819  *      is not found or a pointer to the device. The device has not
820  *      had its reference counter increased so the caller must be careful
821  *      about locking. The caller must hold RCU lock.
822  */
823 
824 struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
825 {
826         struct net_device *dev;
827         struct hlist_head *head = dev_index_hash(net, ifindex);
828 
829         hlist_for_each_entry_rcu(dev, head, index_hlist)
830                 if (dev->ifindex == ifindex)
831                         return dev;
832 
833         return NULL;
834 }
835 EXPORT_SYMBOL(dev_get_by_index_rcu);
836 
837 
838 /**
839  *      dev_get_by_index - find a device by its ifindex
840  *      @net: the applicable net namespace
841  *      @ifindex: index of device
842  *
843  *      Search for an interface by index. Returns NULL if the device
844  *      is not found or a pointer to the device. The device returned has
845  *      had a reference added and the pointer is safe until the user calls
846  *      dev_put to indicate they have finished with it.
847  */
848 
849 struct net_device *dev_get_by_index(struct net *net, int ifindex)
850 {
851         struct net_device *dev;
852 
853         rcu_read_lock();
854         dev = dev_get_by_index_rcu(net, ifindex);
855         if (dev)
856                 dev_hold(dev);
857         rcu_read_unlock();
858         return dev;
859 }
860 EXPORT_SYMBOL(dev_get_by_index);
861 
862 /**
863  *      netdev_get_name - get a netdevice name, knowing its ifindex.
864  *      @net: network namespace
865  *      @name: a pointer to the buffer where the name will be stored.
866  *      @ifindex: the ifindex of the interface to get the name from.
867  *
868  *      The use of raw_seqcount_begin() and cond_resched() before
869  *      retrying is required as we want to give the writers a chance
870  *      to complete when CONFIG_PREEMPT is not set.
871  */
872 int netdev_get_name(struct net *net, char *name, int ifindex)
873 {
874         struct net_device *dev;
875         unsigned int seq;
876 
877 retry:
878         seq = raw_seqcount_begin(&devnet_rename_seq);
879         rcu_read_lock();
880         dev = dev_get_by_index_rcu(net, ifindex);
881         if (!dev) {
882                 rcu_read_unlock();
883                 return -ENODEV;
884         }
885 
886         strcpy(name, dev->name);
887         rcu_read_unlock();
888         if (read_seqcount_retry(&devnet_rename_seq, seq)) {
889                 cond_resched();
890                 goto retry;
891         }
892 
893         return 0;
894 }
895 
896 /**
897  *      dev_getbyhwaddr_rcu - find a device by its hardware address
898  *      @net: the applicable net namespace
899  *      @type: media type of device
900  *      @ha: hardware address
901  *
902  *      Search for an interface by MAC address. Returns NULL if the device
903  *      is not found or a pointer to the device.
904  *      The caller must hold RCU or RTNL.
905  *      The returned device has not had its ref count increased
906  *      and the caller must therefore be careful about locking
907  *
908  */
909 
910 struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
911                                        const char *ha)
912 {
913         struct net_device *dev;
914 
915         for_each_netdev_rcu(net, dev)
916                 if (dev->type == type &&
917                     !memcmp(dev->dev_addr, ha, dev->addr_len))
918                         return dev;
919 
920         return NULL;
921 }
922 EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
923 
924 struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
925 {
926         struct net_device *dev;
927 
928         ASSERT_RTNL();
929         for_each_netdev(net, dev)
930                 if (dev->type == type)
931                         return dev;
932 
933         return NULL;
934 }
935 EXPORT_SYMBOL(__dev_getfirstbyhwtype);
936 
937 struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
938 {
939         struct net_device *dev, *ret = NULL;
940 
941         rcu_read_lock();
942         for_each_netdev_rcu(net, dev)
943                 if (dev->type == type) {
944                         dev_hold(dev);
945                         ret = dev;
946                         break;
947                 }
948         rcu_read_unlock();
949         return ret;
950 }
951 EXPORT_SYMBOL(dev_getfirstbyhwtype);
952 
953 /**
954  *      __dev_get_by_flags - find any device with given flags
955  *      @net: the applicable net namespace
956  *      @if_flags: IFF_* values
957  *      @mask: bitmask of bits in if_flags to check
958  *
959  *      Search for any interface with the given flags. Returns NULL if a device
960  *      is not found or a pointer to the device. Must be called inside
961  *      rtnl_lock(), and result refcount is unchanged.
962  */
963 
964 struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
965                                       unsigned short mask)
966 {
967         struct net_device *dev, *ret;
968 
969         ASSERT_RTNL();
970 
971         ret = NULL;
972         for_each_netdev(net, dev) {
973                 if (((dev->flags ^ if_flags) & mask) == 0) {
974                         ret = dev;
975                         break;
976                 }
977         }
978         return ret;
979 }
980 EXPORT_SYMBOL(__dev_get_by_flags);
981 
982 /**
983  *      dev_valid_name - check if name is okay for network device
984  *      @name: name string
985  *
986  *      Network device names need to be valid file names to
987  *      to allow sysfs to work.  We also disallow any kind of
988  *      whitespace.
989  */
990 bool dev_valid_name(const char *name)
991 {
992         if (*name == '\0')
993                 return false;
994         if (strlen(name) >= IFNAMSIZ)
995                 return false;
996         if (!strcmp(name, ".") || !strcmp(name, ".."))
997                 return false;
998 
999         while (*name) {
1000                 if (*name == '/' || *name == ':' || isspace(*name))
1001                         return false;
1002                 name++;
1003         }
1004         return true;
1005 }
1006 EXPORT_SYMBOL(dev_valid_name);
1007 
1008 /**
1009  *      __dev_alloc_name - allocate a name for a device
1010  *      @net: network namespace to allocate the device name in
1011  *      @name: name format string
1012  *      @buf:  scratch buffer and result name string
1013  *
1014  *      Passed a format string - eg "lt%d" it will try and find a suitable
1015  *      id. It scans list of devices to build up a free map, then chooses
1016  *      the first empty slot. The caller must hold the dev_base or rtnl lock
1017  *      while allocating the name and adding the device in order to avoid
1018  *      duplicates.
1019  *      Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1020  *      Returns the number of the unit assigned or a negative errno code.
1021  */
1022 
1023 static int __dev_alloc_name(struct net *net, const char *name, char *buf)
1024 {
1025         int i = 0;
1026         const char *p;
1027         const int max_netdevices = 8*PAGE_SIZE;
1028         unsigned long *inuse;
1029         struct net_device *d;
1030 
1031         p = strnchr(name, IFNAMSIZ-1, '%');
1032         if (p) {
1033                 /*
1034                  * Verify the string as this thing may have come from
1035                  * the user.  There must be either one "%d" and no other "%"
1036                  * characters.
1037                  */
1038                 if (p[1] != 'd' || strchr(p + 2, '%'))
1039                         return -EINVAL;
1040 
1041                 /* Use one page as a bit array of possible slots */
1042                 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
1043                 if (!inuse)
1044                         return -ENOMEM;
1045 
1046                 for_each_netdev(net, d) {
1047                         if (!sscanf(d->name, name, &i))
1048                                 continue;
1049                         if (i < 0 || i >= max_netdevices)
1050                                 continue;
1051 
1052                         /*  avoid cases where sscanf is not exact inverse of printf */
1053                         snprintf(buf, IFNAMSIZ, name, i);
1054                         if (!strncmp(buf, d->name, IFNAMSIZ))
1055                                 set_bit(i, inuse);
1056                 }
1057 
1058                 i = find_first_zero_bit(inuse, max_netdevices);
1059                 free_page((unsigned long) inuse);
1060         }
1061 
1062         if (buf != name)
1063                 snprintf(buf, IFNAMSIZ, name, i);
1064         if (!__dev_get_by_name(net, buf))
1065                 return i;
1066 
1067         /* It is possible to run out of possible slots
1068          * when the name is long and there isn't enough space left
1069          * for the digits, or if all bits are used.
1070          */
1071         return -ENFILE;
1072 }
1073 
1074 /**
1075  *      dev_alloc_name - allocate a name for a device
1076  *      @dev: device
1077  *      @name: name format string
1078  *
1079  *      Passed a format string - eg "lt%d" it will try and find a suitable
1080  *      id. It scans list of devices to build up a free map, then chooses
1081  *      the first empty slot. The caller must hold the dev_base or rtnl lock
1082  *      while allocating the name and adding the device in order to avoid
1083  *      duplicates.
1084  *      Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1085  *      Returns the number of the unit assigned or a negative errno code.
1086  */
1087 
1088 int dev_alloc_name(struct net_device *dev, const char *name)
1089 {
1090         char buf[IFNAMSIZ];
1091         struct net *net;
1092         int ret;
1093 
1094         BUG_ON(!dev_net(dev));
1095         net = dev_net(dev);
1096         ret = __dev_alloc_name(net, name, buf);
1097         if (ret >= 0)
1098                 strlcpy(dev->name, buf, IFNAMSIZ);
1099         return ret;
1100 }
1101 EXPORT_SYMBOL(dev_alloc_name);
1102 
1103 static int dev_alloc_name_ns(struct net *net,
1104                              struct net_device *dev,
1105                              const char *name)
1106 {
1107         char buf[IFNAMSIZ];
1108         int ret;
1109 
1110         ret = __dev_alloc_name(net, name, buf);
1111         if (ret >= 0)
1112                 strlcpy(dev->name, buf, IFNAMSIZ);
1113         return ret;
1114 }
1115 
1116 static int dev_get_valid_name(struct net *net,
1117                               struct net_device *dev,
1118                               const char *name)
1119 {
1120         BUG_ON(!net);
1121 
1122         if (!dev_valid_name(name))
1123                 return -EINVAL;
1124 
1125         if (strchr(name, '%'))
1126                 return dev_alloc_name_ns(net, dev, name);
1127         else if (__dev_get_by_name(net, name))
1128                 return -EEXIST;
1129         else if (dev->name != name)
1130                 strlcpy(dev->name, name, IFNAMSIZ);
1131 
1132         return 0;
1133 }
1134 
1135 /**
1136  *      dev_change_name - change name of a device
1137  *      @dev: device
1138  *      @newname: name (or format string) must be at least IFNAMSIZ
1139  *
1140  *      Change name of a device, can pass format strings "eth%d".
1141  *      for wildcarding.
1142  */
1143 int dev_change_name(struct net_device *dev, const char *newname)
1144 {
1145         unsigned char old_assign_type;
1146         char oldname[IFNAMSIZ];
1147         int err = 0;
1148         int ret;
1149         struct net *net;
1150 
1151         ASSERT_RTNL();
1152         BUG_ON(!dev_net(dev));
1153 
1154         net = dev_net(dev);
1155         if (dev->flags & IFF_UP)
1156                 return -EBUSY;
1157 
1158         write_seqcount_begin(&devnet_rename_seq);
1159 
1160         if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
1161                 write_seqcount_end(&devnet_rename_seq);
1162                 return 0;
1163         }
1164 
1165         memcpy(oldname, dev->name, IFNAMSIZ);
1166 
1167         err = dev_get_valid_name(net, dev, newname);
1168         if (err < 0) {
1169                 write_seqcount_end(&devnet_rename_seq);
1170                 return err;
1171         }
1172 
1173         if (oldname[0] && !strchr(oldname, '%'))
1174                 netdev_info(dev, "renamed from %s\n", oldname);
1175 
1176         old_assign_type = dev->name_assign_type;
1177         dev->name_assign_type = NET_NAME_RENAMED;
1178 
1179 rollback:
1180         ret = device_rename(&dev->dev, dev->name);
1181         if (ret) {
1182                 memcpy(dev->name, oldname, IFNAMSIZ);
1183                 dev->name_assign_type = old_assign_type;
1184                 write_seqcount_end(&devnet_rename_seq);
1185                 return ret;
1186         }
1187 
1188         write_seqcount_end(&devnet_rename_seq);
1189 
1190         netdev_adjacent_rename_links(dev, oldname);
1191 
1192         write_lock_bh(&dev_base_lock);
1193         hlist_del_rcu(&dev->name_hlist);
1194         write_unlock_bh(&dev_base_lock);
1195 
1196         synchronize_rcu();
1197 
1198         write_lock_bh(&dev_base_lock);
1199         hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
1200         write_unlock_bh(&dev_base_lock);
1201 
1202         ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
1203         ret = notifier_to_errno(ret);
1204 
1205         if (ret) {
1206                 /* err >= 0 after dev_alloc_name() or stores the first errno */
1207                 if (err >= 0) {
1208                         err = ret;
1209                         write_seqcount_begin(&devnet_rename_seq);
1210                         memcpy(dev->name, oldname, IFNAMSIZ);
1211                         memcpy(oldname, newname, IFNAMSIZ);
1212                         dev->name_assign_type = old_assign_type;
1213                         old_assign_type = NET_NAME_RENAMED;
1214                         goto rollback;
1215                 } else {
1216                         pr_err("%s: name change rollback failed: %d\n",
1217                                dev->name, ret);
1218                 }
1219         }
1220 
1221         return err;
1222 }
1223 
1224 /**
1225  *      dev_set_alias - change ifalias of a device
1226  *      @dev: device
1227  *      @alias: name up to IFALIASZ
1228  *      @len: limit of bytes to copy from info
1229  *
1230  *      Set ifalias for a device,
1231  */
1232 int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1233 {
1234         char *new_ifalias;
1235 
1236         ASSERT_RTNL();
1237 
1238         if (len >= IFALIASZ)
1239                 return -EINVAL;
1240 
1241         if (!len) {
1242                 kfree(dev->ifalias);
1243                 dev->ifalias = NULL;
1244                 return 0;
1245         }
1246 
1247         new_ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
1248         if (!new_ifalias)
1249                 return -ENOMEM;
1250         dev->ifalias = new_ifalias;
1251 
1252         strlcpy(dev->ifalias, alias, len+1);
1253         return len;
1254 }
1255 
1256 
1257 /**
1258  *      netdev_features_change - device changes features
1259  *      @dev: device to cause notification
1260  *
1261  *      Called to indicate a device has changed features.
1262  */
1263 void netdev_features_change(struct net_device *dev)
1264 {
1265         call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
1266 }
1267 EXPORT_SYMBOL(netdev_features_change);
1268 
1269 /**
1270  *      netdev_state_change - device changes state
1271  *      @dev: device to cause notification
1272  *
1273  *      Called to indicate a device has changed state. This function calls
1274  *      the notifier chains for netdev_chain and sends a NEWLINK message
1275  *      to the routing socket.
1276  */
1277 void netdev_state_change(struct net_device *dev)
1278 {
1279         if (dev->flags & IFF_UP) {
1280                 struct netdev_notifier_change_info change_info;
1281 
1282                 change_info.flags_changed = 0;
1283                 call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
1284                                               &change_info.info);
1285                 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL);
1286         }
1287 }
1288 EXPORT_SYMBOL(netdev_state_change);
1289 
1290 /**
1291  *      netdev_notify_peers - notify network peers about existence of @dev
1292  *      @dev: network device
1293  *
1294  * Generate traffic such that interested network peers are aware of
1295  * @dev, such as by generating a gratuitous ARP. This may be used when
1296  * a device wants to inform the rest of the network about some sort of
1297  * reconfiguration such as a failover event or virtual machine
1298  * migration.
1299  */
1300 void netdev_notify_peers(struct net_device *dev)
1301 {
1302         rtnl_lock();
1303         call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1304         rtnl_unlock();
1305 }
1306 EXPORT_SYMBOL(netdev_notify_peers);
1307 
1308 static int __dev_open(struct net_device *dev)
1309 {
1310         const struct net_device_ops *ops = dev->netdev_ops;
1311         int ret;
1312 
1313         ASSERT_RTNL();
1314 
1315         if (!netif_device_present(dev))
1316                 return -ENODEV;
1317 
1318         /* Block netpoll from trying to do any rx path servicing.
1319          * If we don't do this there is a chance ndo_poll_controller
1320          * or ndo_poll may be running while we open the device
1321          */
1322         netpoll_poll_disable(dev);
1323 
1324         ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1325         ret = notifier_to_errno(ret);
1326         if (ret)
1327                 return ret;
1328 
1329         set_bit(__LINK_STATE_START, &dev->state);
1330 
1331         if (ops->ndo_validate_addr)
1332                 ret = ops->ndo_validate_addr(dev);
1333 
1334         if (!ret && ops->ndo_open)
1335                 ret = ops->ndo_open(dev);
1336 
1337         netpoll_poll_enable(dev);
1338 
1339         if (ret)
1340                 clear_bit(__LINK_STATE_START, &dev->state);
1341         else {
1342                 dev->flags |= IFF_UP;
1343                 dev_set_rx_mode(dev);
1344                 dev_activate(dev);
1345                 add_device_randomness(dev->dev_addr, dev->addr_len);
1346         }
1347 
1348         return ret;
1349 }
1350 
1351 /**
1352  *      dev_open        - prepare an interface for use.
1353  *      @dev:   device to open
1354  *
1355  *      Takes a device from down to up state. The device's private open
1356  *      function is invoked and then the multicast lists are loaded. Finally
1357  *      the device is moved into the up state and a %NETDEV_UP message is
1358  *      sent to the netdev notifier chain.
1359  *
1360  *      Calling this function on an active interface is a nop. On a failure
1361  *      a negative errno code is returned.
1362  */
1363 int dev_open(struct net_device *dev)
1364 {
1365         int ret;
1366 
1367         if (dev->flags & IFF_UP)
1368                 return 0;
1369 
1370         ret = __dev_open(dev);
1371         if (ret < 0)
1372                 return ret;
1373 
1374         rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1375         call_netdevice_notifiers(NETDEV_UP, dev);
1376 
1377         return ret;
1378 }
1379 EXPORT_SYMBOL(dev_open);
1380 
1381 static int __dev_close_many(struct list_head *head)
1382 {
1383         struct net_device *dev;
1384 
1385         ASSERT_RTNL();
1386         might_sleep();
1387 
1388         list_for_each_entry(dev, head, close_list) {
1389                 /* Temporarily disable netpoll until the interface is down */
1390                 netpoll_poll_disable(dev);
1391 
1392                 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1393 
1394                 clear_bit(__LINK_STATE_START, &dev->state);
1395 
1396                 /* Synchronize to scheduled poll. We cannot touch poll list, it
1397                  * can be even on different cpu. So just clear netif_running().
1398                  *
1399                  * dev->stop() will invoke napi_disable() on all of it's
1400                  * napi_struct instances on this device.
1401                  */
1402                 smp_mb__after_atomic(); /* Commit netif_running(). */
1403         }
1404 
1405         dev_deactivate_many(head);
1406 
1407         list_for_each_entry(dev, head, close_list) {
1408                 const struct net_device_ops *ops = dev->netdev_ops;
1409 
1410                 /*
1411                  *      Call the device specific close. This cannot fail.
1412                  *      Only if device is UP
1413                  *
1414                  *      We allow it to be called even after a DETACH hot-plug
1415                  *      event.
1416                  */
1417                 if (ops->ndo_stop)
1418                         ops->ndo_stop(dev);
1419 
1420                 dev->flags &= ~IFF_UP;
1421                 netpoll_poll_enable(dev);
1422         }
1423 
1424         return 0;
1425 }
1426 
1427 static int __dev_close(struct net_device *dev)
1428 {
1429         int retval;
1430         LIST_HEAD(single);
1431 
1432         list_add(&dev->close_list, &single);
1433         retval = __dev_close_many(&single);
1434         list_del(&single);
1435 
1436         return retval;
1437 }
1438 
1439 int dev_close_many(struct list_head *head, bool unlink)
1440 {
1441         struct net_device *dev, *tmp;
1442 
1443         /* Remove the devices that don't need to be closed */
1444         list_for_each_entry_safe(dev, tmp, head, close_list)
1445                 if (!(dev->flags & IFF_UP))
1446                         list_del_init(&dev->close_list);
1447 
1448         __dev_close_many(head);
1449 
1450         list_for_each_entry_safe(dev, tmp, head, close_list) {
1451                 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1452                 call_netdevice_notifiers(NETDEV_DOWN, dev);
1453                 if (unlink)
1454                         list_del_init(&dev->close_list);
1455         }
1456 
1457         return 0;
1458 }
1459 EXPORT_SYMBOL(dev_close_many);
1460 
1461 /**
1462  *      dev_close - shutdown an interface.
1463  *      @dev: device to shutdown
1464  *
1465  *      This function moves an active device into down state. A
1466  *      %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1467  *      is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1468  *      chain.
1469  */
1470 int dev_close(struct net_device *dev)
1471 {
1472         if (dev->flags & IFF_UP) {
1473                 LIST_HEAD(single);
1474 
1475                 list_add(&dev->close_list, &single);
1476                 dev_close_many(&single, true);
1477                 list_del(&single);
1478         }
1479         return 0;
1480 }
1481 EXPORT_SYMBOL(dev_close);
1482 
1483 
1484 /**
1485  *      dev_disable_lro - disable Large Receive Offload on a device
1486  *      @dev: device
1487  *
1488  *      Disable Large Receive Offload (LRO) on a net device.  Must be
1489  *      called under RTNL.  This is needed if received packets may be
1490  *      forwarded to another interface.
1491  */
1492 void dev_disable_lro(struct net_device *dev)
1493 {
1494         struct net_device *lower_dev;
1495         struct list_head *iter;
1496 
1497         dev->wanted_features &= ~NETIF_F_LRO;
1498         netdev_update_features(dev);
1499 
1500         if (unlikely(dev->features & NETIF_F_LRO))
1501                 netdev_WARN(dev, "failed to disable LRO!\n");
1502 
1503         netdev_for_each_lower_dev(dev, lower_dev, iter)
1504                 dev_disable_lro(lower_dev);
1505 }
1506 EXPORT_SYMBOL(dev_disable_lro);
1507 
1508 static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1509                                    struct net_device *dev)
1510 {
1511         struct netdev_notifier_info info;
1512 
1513         netdev_notifier_info_init(&info, dev);
1514         return nb->notifier_call(nb, val, &info);
1515 }
1516 
1517 static int dev_boot_phase = 1;
1518 
1519 /**
1520  *      register_netdevice_notifier - register a network notifier block
1521  *      @nb: notifier
1522  *
1523  *      Register a notifier to be called when network device events occur.
1524  *      The notifier passed is linked into the kernel structures and must
1525  *      not be reused until it has been unregistered. A negative errno code
1526  *      is returned on a failure.
1527  *
1528  *      When registered all registration and up events are replayed
1529  *      to the new notifier to allow device to have a race free
1530  *      view of the network device list.
1531  */
1532 
1533 int register_netdevice_notifier(struct notifier_block *nb)
1534 {
1535         struct net_device *dev;
1536         struct net_device *last;
1537         struct net *net;
1538         int err;
1539 
1540         rtnl_lock();
1541         err = raw_notifier_chain_register(&netdev_chain, nb);
1542         if (err)
1543                 goto unlock;
1544         if (dev_boot_phase)
1545                 goto unlock;
1546         for_each_net(net) {
1547                 for_each_netdev(net, dev) {
1548                         err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
1549                         err = notifier_to_errno(err);
1550                         if (err)
1551                                 goto rollback;
1552 
1553                         if (!(dev->flags & IFF_UP))
1554                                 continue;
1555 
1556                         call_netdevice_notifier(nb, NETDEV_UP, dev);
1557                 }
1558         }
1559 
1560 unlock:
1561         rtnl_unlock();
1562         return err;
1563 
1564 rollback:
1565         last = dev;
1566         for_each_net(net) {
1567                 for_each_netdev(net, dev) {
1568                         if (dev == last)
1569                                 goto outroll;
1570 
1571                         if (dev->flags & IFF_UP) {
1572                                 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1573                                                         dev);
1574                                 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1575                         }
1576                         call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1577                 }
1578         }
1579 
1580 outroll:
1581         raw_notifier_chain_unregister(&netdev_chain, nb);
1582         goto unlock;
1583 }
1584 EXPORT_SYMBOL(register_netdevice_notifier);
1585 
1586 /**
1587  *      unregister_netdevice_notifier - unregister a network notifier block
1588  *      @nb: notifier
1589  *
1590  *      Unregister a notifier previously registered by
1591  *      register_netdevice_notifier(). The notifier is unlinked into the
1592  *      kernel structures and may then be reused. A negative errno code
1593  *      is returned on a failure.
1594  *
1595  *      After unregistering unregister and down device events are synthesized
1596  *      for all devices on the device list to the removed notifier to remove
1597  *      the need for special case cleanup code.
1598  */
1599 
1600 int unregister_netdevice_notifier(struct notifier_block *nb)
1601 {
1602         struct net_device *dev;
1603         struct net *net;
1604         int err;
1605 
1606         rtnl_lock();
1607         err = raw_notifier_chain_unregister(&netdev_chain, nb);
1608         if (err)
1609                 goto unlock;
1610 
1611         for_each_net(net) {
1612                 for_each_netdev(net, dev) {
1613                         if (dev->flags & IFF_UP) {
1614                                 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1615                                                         dev);
1616                                 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1617                         }
1618                         call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1619                 }
1620         }
1621 unlock:
1622         rtnl_unlock();
1623         return err;
1624 }
1625 EXPORT_SYMBOL(unregister_netdevice_notifier);
1626 
1627 /**
1628  *      call_netdevice_notifiers_info - call all network notifier blocks
1629  *      @val: value passed unmodified to notifier function
1630  *      @dev: net_device pointer passed unmodified to notifier function
1631  *      @info: notifier information data
1632  *
1633  *      Call all network notifier blocks.  Parameters and return value
1634  *      are as for raw_notifier_call_chain().
1635  */
1636 
1637 static int call_netdevice_notifiers_info(unsigned long val,
1638                                          struct net_device *dev,
1639                                          struct netdev_notifier_info *info)
1640 {
1641         ASSERT_RTNL();
1642         netdev_notifier_info_init(info, dev);
1643         return raw_notifier_call_chain(&netdev_chain, val, info);
1644 }
1645 
1646 /**
1647  *      call_netdevice_notifiers - call all network notifier blocks
1648  *      @val: value passed unmodified to notifier function
1649  *      @dev: net_device pointer passed unmodified to notifier function
1650  *
1651  *      Call all network notifier blocks.  Parameters and return value
1652  *      are as for raw_notifier_call_chain().
1653  */
1654 
1655 int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
1656 {
1657         struct netdev_notifier_info info;
1658 
1659         return call_netdevice_notifiers_info(val, dev, &info);
1660 }
1661 EXPORT_SYMBOL(call_netdevice_notifiers);
1662 
1663 #ifdef CONFIG_NET_INGRESS
1664 static struct static_key ingress_needed __read_mostly;
1665 
1666 void net_inc_ingress_queue(void)
1667 {
1668         static_key_slow_inc(&ingress_needed);
1669 }
1670 EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
1671 
1672 void net_dec_ingress_queue(void)
1673 {
1674         static_key_slow_dec(&ingress_needed);
1675 }
1676 EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
1677 #endif
1678 
1679 #ifdef CONFIG_NET_EGRESS
1680 static struct static_key egress_needed __read_mostly;
1681 
1682 void net_inc_egress_queue(void)
1683 {
1684         static_key_slow_inc(&egress_needed);
1685 }
1686 EXPORT_SYMBOL_GPL(net_inc_egress_queue);
1687 
1688 void net_dec_egress_queue(void)
1689 {
1690         static_key_slow_dec(&egress_needed);
1691 }
1692 EXPORT_SYMBOL_GPL(net_dec_egress_queue);
1693 #endif
1694 
1695 static struct static_key netstamp_needed __read_mostly;
1696 #ifdef HAVE_JUMP_LABEL
1697 /* We are not allowed to call static_key_slow_dec() from irq context
1698  * If net_disable_timestamp() is called from irq context, defer the
1699  * static_key_slow_dec() calls.
1700  */
1701 static atomic_t netstamp_needed_deferred;
1702 #endif
1703 
1704 void net_enable_timestamp(void)
1705 {
1706 #ifdef HAVE_JUMP_LABEL
1707         int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
1708 
1709         if (deferred) {
1710                 while (--deferred)
1711                         static_key_slow_dec(&netstamp_needed);
1712                 return;
1713         }
1714 #endif
1715         static_key_slow_inc(&netstamp_needed);
1716 }
1717 EXPORT_SYMBOL(net_enable_timestamp);
1718 
1719 void net_disable_timestamp(void)
1720 {
1721 #ifdef HAVE_JUMP_LABEL
1722         if (in_interrupt()) {
1723                 atomic_inc(&netstamp_needed_deferred);
1724                 return;
1725         }
1726 #endif
1727         static_key_slow_dec(&netstamp_needed);
1728 }
1729 EXPORT_SYMBOL(net_disable_timestamp);
1730 
1731 static inline void net_timestamp_set(struct sk_buff *skb)
1732 {
1733         skb->tstamp.tv64 = 0;
1734         if (static_key_false(&netstamp_needed))
1735                 __net_timestamp(skb);
1736 }
1737 
1738 #define net_timestamp_check(COND, SKB)                  \
1739         if (static_key_false(&netstamp_needed)) {               \
1740                 if ((COND) && !(SKB)->tstamp.tv64)      \
1741                         __net_timestamp(SKB);           \
1742         }                                               \
1743 
1744 bool is_skb_forwardable(struct net_device *dev, struct sk_buff *skb)
1745 {
1746         unsigned int len;
1747 
1748         if (!(dev->flags & IFF_UP))
1749                 return false;
1750 
1751         len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
1752         if (skb->len <= len)
1753                 return true;
1754 
1755         /* if TSO is enabled, we don't care about the length as the packet
1756          * could be forwarded without being segmented before
1757          */
1758         if (skb_is_gso(skb))
1759                 return true;
1760 
1761         return false;
1762 }
1763 EXPORT_SYMBOL_GPL(is_skb_forwardable);
1764 
1765 int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1766 {
1767         if (skb_orphan_frags(skb, GFP_ATOMIC) ||
1768             unlikely(!is_skb_forwardable(dev, skb))) {
1769                 atomic_long_inc(&dev->rx_dropped);
1770                 kfree_skb(skb);
1771                 return NET_RX_DROP;
1772         }
1773 
1774         skb_scrub_packet(skb, true);
1775         skb->priority = 0;
1776         skb->protocol = eth_type_trans(skb, dev);
1777         skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
1778 
1779         return 0;
1780 }
1781 EXPORT_SYMBOL_GPL(__dev_forward_skb);
1782 
1783 /**
1784  * dev_forward_skb - loopback an skb to another netif
1785  *
1786  * @dev: destination network device
1787  * @skb: buffer to forward
1788  *
1789  * return values:
1790  *      NET_RX_SUCCESS  (no congestion)
1791  *      NET_RX_DROP     (packet was dropped, but freed)
1792  *
1793  * dev_forward_skb can be used for injecting an skb from the
1794  * start_xmit function of one device into the receive queue
1795  * of another device.
1796  *
1797  * The receiving device may be in another namespace, so
1798  * we have to clear all information in the skb that could
1799  * impact namespace isolation.
1800  */
1801 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1802 {
1803         return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
1804 }
1805 EXPORT_SYMBOL_GPL(dev_forward_skb);
1806 
1807 static inline int deliver_skb(struct sk_buff *skb,
1808                               struct packet_type *pt_prev,
1809                               struct net_device *orig_dev)
1810 {
1811         if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
1812                 return -ENOMEM;
1813         atomic_inc(&skb->users);
1814         return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1815 }
1816 
1817 static inline void deliver_ptype_list_skb(struct sk_buff *skb,
1818                                           struct packet_type **pt,
1819                                           struct net_device *orig_dev,
1820                                           __be16 type,
1821                                           struct list_head *ptype_list)
1822 {
1823         struct packet_type *ptype, *pt_prev = *pt;
1824 
1825         list_for_each_entry_rcu(ptype, ptype_list, list) {
1826                 if (ptype->type != type)
1827                         continue;
1828                 if (pt_prev)
1829                         deliver_skb(skb, pt_prev, orig_dev);
1830                 pt_prev = ptype;
1831         }
1832         *pt = pt_prev;
1833 }
1834 
1835 static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
1836 {
1837         if (!ptype->af_packet_priv || !skb->sk)
1838                 return false;
1839 
1840         if (ptype->id_match)
1841                 return ptype->id_match(ptype, skb->sk);
1842         else if ((struct sock *)ptype->af_packet_priv == skb->sk)
1843                 return true;
1844 
1845         return false;
1846 }
1847 
1848 /*
1849  *      Support routine. Sends outgoing frames to any network
1850  *      taps currently in use.
1851  */
1852 
1853 static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
1854 {
1855         struct packet_type *ptype;
1856         struct sk_buff *skb2 = NULL;
1857         struct packet_type *pt_prev = NULL;
1858         struct list_head *ptype_list = &ptype_all;
1859 
1860         rcu_read_lock();
1861 again:
1862         list_for_each_entry_rcu(ptype, ptype_list, list) {
1863                 /* Never send packets back to the socket
1864                  * they originated from - MvS (miquels@drinkel.ow.org)
1865                  */
1866                 if (skb_loop_sk(ptype, skb))
1867                         continue;
1868 
1869                 if (pt_prev) {
1870                         deliver_skb(skb2, pt_prev, skb->dev);
1871                         pt_prev = ptype;
1872                         continue;
1873                 }
1874 
1875                 /* need to clone skb, done only once */
1876                 skb2 = skb_clone(skb, GFP_ATOMIC);
1877                 if (!skb2)
1878                         goto out_unlock;
1879 
1880                 net_timestamp_set(skb2);
1881 
1882                 /* skb->nh should be correctly
1883                  * set by sender, so that the second statement is
1884                  * just protection against buggy protocols.
1885                  */
1886                 skb_reset_mac_header(skb2);
1887 
1888                 if (skb_network_header(skb2) < skb2->data ||
1889                     skb_network_header(skb2) > skb_tail_pointer(skb2)) {
1890                         net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
1891                                              ntohs(skb2->protocol),
1892                                              dev->name);
1893                         skb_reset_network_header(skb2);
1894                 }
1895 
1896                 skb2->transport_header = skb2->network_header;
1897                 skb2->pkt_type = PACKET_OUTGOING;
1898                 pt_prev = ptype;
1899         }
1900 
1901         if (ptype_list == &ptype_all) {
1902                 ptype_list = &dev->ptype_all;
1903                 goto again;
1904         }
1905 out_unlock:
1906         if (pt_prev)
1907                 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
1908         rcu_read_unlock();
1909 }
1910 
1911 /**
1912  * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
1913  * @dev: Network device
1914  * @txq: number of queues available
1915  *
1916  * If real_num_tx_queues is changed the tc mappings may no longer be
1917  * valid. To resolve this verify the tc mapping remains valid and if
1918  * not NULL the mapping. With no priorities mapping to this
1919  * offset/count pair it will no longer be used. In the worst case TC0
1920  * is invalid nothing can be done so disable priority mappings. If is
1921  * expected that drivers will fix this mapping if they can before
1922  * calling netif_set_real_num_tx_queues.
1923  */
1924 static void netif_setup_tc(struct net_device *dev, unsigned int txq)
1925 {
1926         int i;
1927         struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
1928 
1929         /* If TC0 is invalidated disable TC mapping */
1930         if (tc->offset + tc->count > txq) {
1931                 pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
1932                 dev->num_tc = 0;
1933                 return;
1934         }
1935 
1936         /* Invalidated prio to tc mappings set to TC0 */
1937         for (i = 1; i < TC_BITMASK + 1; i++) {
1938                 int q = netdev_get_prio_tc_map(dev, i);
1939 
1940                 tc = &dev->tc_to_txq[q];
1941                 if (tc->offset + tc->count > txq) {
1942                         pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
1943                                 i, q);
1944                         netdev_set_prio_tc_map(dev, i, 0);
1945                 }
1946         }
1947 }
1948 
1949 #ifdef CONFIG_XPS
1950 static DEFINE_MUTEX(xps_map_mutex);
1951 #define xmap_dereference(P)             \
1952         rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
1953 
1954 static struct xps_map *remove_xps_queue(struct xps_dev_maps *dev_maps,
1955                                         int cpu, u16 index)
1956 {
1957         struct xps_map *map = NULL;
1958         int pos;
1959 
1960         if (dev_maps)
1961                 map = xmap_dereference(dev_maps->cpu_map[cpu]);
1962 
1963         for (pos = 0; map && pos < map->len; pos++) {
1964                 if (map->queues[pos] == index) {
1965                         if (map->len > 1) {
1966                                 map->queues[pos] = map->queues[--map->len];
1967                         } else {
1968                                 RCU_INIT_POINTER(dev_maps->cpu_map[cpu], NULL);
1969                                 kfree_rcu(map, rcu);
1970                                 map = NULL;
1971                         }
1972                         break;
1973                 }
1974         }
1975 
1976         return map;
1977 }
1978 
1979 static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
1980 {
1981         struct xps_dev_maps *dev_maps;
1982         int cpu, i;
1983         bool active = false;
1984 
1985         mutex_lock(&xps_map_mutex);
1986         dev_maps = xmap_dereference(dev->xps_maps);
1987 
1988         if (!dev_maps)
1989                 goto out_no_maps;
1990 
1991         for_each_possible_cpu(cpu) {
1992                 for (i = index; i < dev->num_tx_queues; i++) {
1993                         if (!remove_xps_queue(dev_maps, cpu, i))
1994                                 break;
1995                 }
1996                 if (i == dev->num_tx_queues)
1997                         active = true;
1998         }
1999 
2000         if (!active) {
2001                 RCU_INIT_POINTER(dev->xps_maps, NULL);
2002                 kfree_rcu(dev_maps, rcu);
2003         }
2004 
2005         for (i = index; i < dev->num_tx_queues; i++)
2006                 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i),
2007                                              NUMA_NO_NODE);
2008 
2009 out_no_maps:
2010         mutex_unlock(&xps_map_mutex);
2011 }
2012 
2013 static struct xps_map *expand_xps_map(struct xps_map *map,
2014                                       int cpu, u16 index)
2015 {
2016         struct xps_map *new_map;
2017         int alloc_len = XPS_MIN_MAP_ALLOC;
2018         int i, pos;
2019 
2020         for (pos = 0; map && pos < map->len; pos++) {
2021                 if (map->queues[pos] != index)
2022                         continue;
2023                 return map;
2024         }
2025 
2026         /* Need to add queue to this CPU's existing map */
2027         if (map) {
2028                 if (pos < map->alloc_len)
2029                         return map;
2030 
2031                 alloc_len = map->alloc_len * 2;
2032         }
2033 
2034         /* Need to allocate new map to store queue on this CPU's map */
2035         new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
2036                                cpu_to_node(cpu));
2037         if (!new_map)
2038                 return NULL;
2039 
2040         for (i = 0; i < pos; i++)
2041                 new_map->queues[i] = map->queues[i];
2042         new_map->alloc_len = alloc_len;
2043         new_map->len = pos;
2044 
2045         return new_map;
2046 }
2047 
2048 int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2049                         u16 index)
2050 {
2051         struct xps_dev_maps *dev_maps, *new_dev_maps = NULL;
2052         struct xps_map *map, *new_map;
2053         int maps_sz = max_t(unsigned int, XPS_DEV_MAPS_SIZE, L1_CACHE_BYTES);
2054         int cpu, numa_node_id = -2;
2055         bool active = false;
2056 
2057         mutex_lock(&xps_map_mutex);
2058 
2059         dev_maps = xmap_dereference(dev->xps_maps);
2060 
2061         /* allocate memory for queue storage */
2062         for_each_online_cpu(cpu) {
2063                 if (!cpumask_test_cpu(cpu, mask))
2064                         continue;
2065 
2066                 if (!new_dev_maps)
2067                         new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
2068                 if (!new_dev_maps) {
2069                         mutex_unlock(&xps_map_mutex);
2070                         return -ENOMEM;
2071                 }
2072 
2073                 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) :
2074                                  NULL;
2075 
2076                 map = expand_xps_map(map, cpu, index);
2077                 if (!map)
2078                         goto error;
2079 
2080                 RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map);
2081         }
2082 
2083         if (!new_dev_maps)
2084                 goto out_no_new_maps;
2085 
2086         for_each_possible_cpu(cpu) {
2087                 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) {
2088                         /* add queue to CPU maps */
2089                         int pos = 0;
2090 
2091                         map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
2092                         while ((pos < map->len) && (map->queues[pos] != index))
2093                                 pos++;
2094 
2095                         if (pos == map->len)
2096                                 map->queues[map->len++] = index;
2097 #ifdef CONFIG_NUMA
2098                         if (numa_node_id == -2)
2099                                 numa_node_id = cpu_to_node(cpu);
2100                         else if (numa_node_id != cpu_to_node(cpu))
2101                                 numa_node_id = -1;
2102 #endif
2103                 } else if (dev_maps) {
2104                         /* fill in the new device map from the old device map */
2105                         map = xmap_dereference(dev_maps->cpu_map[cpu]);
2106                         RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map);
2107                 }
2108 
2109         }
2110 
2111         rcu_assign_pointer(dev->xps_maps, new_dev_maps);
2112 
2113         /* Cleanup old maps */
2114         if (dev_maps) {
2115                 for_each_possible_cpu(cpu) {
2116                         new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
2117                         map = xmap_dereference(dev_maps->cpu_map[cpu]);
2118                         if (map && map != new_map)
2119                                 kfree_rcu(map, rcu);
2120                 }
2121 
2122                 kfree_rcu(dev_maps, rcu);
2123         }
2124 
2125         dev_maps = new_dev_maps;
2126         active = true;
2127 
2128 out_no_new_maps:
2129         /* update Tx queue numa node */
2130         netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2131                                      (numa_node_id >= 0) ? numa_node_id :
2132                                      NUMA_NO_NODE);
2133 
2134         if (!dev_maps)
2135                 goto out_no_maps;
2136 
2137         /* removes queue from unused CPUs */
2138         for_each_possible_cpu(cpu) {
2139                 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu))
2140                         continue;
2141 
2142                 if (remove_xps_queue(dev_maps, cpu, index))
2143                         active = true;
2144         }
2145 
2146         /* free map if not active */
2147         if (!active) {
2148                 RCU_INIT_POINTER(dev->xps_maps, NULL);
2149                 kfree_rcu(dev_maps, rcu);
2150         }
2151 
2152 out_no_maps:
2153         mutex_unlock(&xps_map_mutex);
2154 
2155         return 0;
2156 error:
2157         /* remove any maps that we added */
2158         for_each_possible_cpu(cpu) {
2159                 new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
2160                 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) :
2161                                  NULL;
2162                 if (new_map && new_map != map)
2163                         kfree(new_map);
2164         }
2165 
2166         mutex_unlock(&xps_map_mutex);
2167 
2168         kfree(new_dev_maps);
2169         return -ENOMEM;
2170 }
2171 EXPORT_SYMBOL(netif_set_xps_queue);
2172 
2173 #endif
2174 /*
2175  * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
2176  * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
2177  */
2178 int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
2179 {
2180         int rc;
2181 
2182         if (txq < 1 || txq > dev->num_tx_queues)
2183                 return -EINVAL;
2184 
2185         if (dev->reg_state == NETREG_REGISTERED ||
2186             dev->reg_state == NETREG_UNREGISTERING) {
2187                 ASSERT_RTNL();
2188 
2189                 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
2190                                                   txq);
2191                 if (rc)
2192                         return rc;
2193 
2194                 if (dev->num_tc)
2195                         netif_setup_tc(dev, txq);
2196 
2197                 if (txq < dev->real_num_tx_queues) {
2198                         qdisc_reset_all_tx_gt(dev, txq);
2199 #ifdef CONFIG_XPS
2200                         netif_reset_xps_queues_gt(dev, txq);
2201 #endif
2202                 }
2203         }
2204 
2205         dev->real_num_tx_queues = txq;
2206         return 0;
2207 }
2208 EXPORT_SYMBOL(netif_set_real_num_tx_queues);
2209 
2210 #ifdef CONFIG_SYSFS
2211 /**
2212  *      netif_set_real_num_rx_queues - set actual number of RX queues used
2213  *      @dev: Network device
2214  *      @rxq: Actual number of RX queues
2215  *
2216  *      This must be called either with the rtnl_lock held or before
2217  *      registration of the net device.  Returns 0 on success, or a
2218  *      negative error code.  If called before registration, it always
2219  *      succeeds.
2220  */
2221 int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
2222 {
2223         int rc;
2224 
2225         if (rxq < 1 || rxq > dev->num_rx_queues)
2226                 return -EINVAL;
2227 
2228         if (dev->reg_state == NETREG_REGISTERED) {
2229                 ASSERT_RTNL();
2230 
2231                 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
2232                                                   rxq);
2233                 if (rc)
2234                         return rc;
2235         }
2236 
2237         dev->real_num_rx_queues = rxq;
2238         return 0;
2239 }
2240 EXPORT_SYMBOL(netif_set_real_num_rx_queues);
2241 #endif
2242 
2243 /**
2244  * netif_get_num_default_rss_queues - default number of RSS queues
2245  *
2246  * This routine should set an upper limit on the number of RSS queues
2247  * used by default by multiqueue devices.
2248  */
2249 int netif_get_num_default_rss_queues(void)
2250 {
2251         return min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
2252 }
2253 EXPORT_SYMBOL(netif_get_num_default_rss_queues);
2254 
2255 static inline void __netif_reschedule(struct Qdisc *q)
2256 {
2257         struct softnet_data *sd;
2258         unsigned long flags;
2259 
2260         local_irq_save(flags);
2261         sd = this_cpu_ptr(&softnet_data);
2262         q->next_sched = NULL;
2263         *sd->output_queue_tailp = q;
2264         sd->output_queue_tailp = &q->next_sched;
2265         raise_softirq_irqoff(NET_TX_SOFTIRQ);
2266         local_irq_restore(flags);
2267 }
2268 
2269 void __netif_schedule(struct Qdisc *q)
2270 {
2271         if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
2272                 __netif_reschedule(q);
2273 }
2274 EXPORT_SYMBOL(__netif_schedule);
2275 
2276 struct dev_kfree_skb_cb {
2277         enum skb_free_reason reason;
2278 };
2279 
2280 static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
2281 {
2282         return (struct dev_kfree_skb_cb *)skb->cb;
2283 }
2284 
2285 void netif_schedule_queue(struct netdev_queue *txq)
2286 {
2287         rcu_read_lock();
2288         if (!(txq->state & QUEUE_STATE_ANY_XOFF)) {
2289                 struct Qdisc *q = rcu_dereference(txq->qdisc);
2290 
2291                 __netif_schedule(q);
2292         }
2293         rcu_read_unlock();
2294 }
2295 EXPORT_SYMBOL(netif_schedule_queue);
2296 
2297 /**
2298  *      netif_wake_subqueue - allow sending packets on subqueue
2299  *      @dev: network device
2300  *      @queue_index: sub queue index
2301  *
2302  * Resume individual transmit queue of a device with multiple transmit queues.
2303  */
2304 void netif_wake_subqueue(struct net_device *dev, u16 queue_index)
2305 {
2306         struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
2307 
2308         if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &txq->state)) {
2309                 struct Qdisc *q;
2310 
2311                 rcu_read_lock();
2312                 q = rcu_dereference(txq->qdisc);
2313                 __netif_schedule(q);
2314                 rcu_read_unlock();
2315         }
2316 }
2317 EXPORT_SYMBOL(netif_wake_subqueue);
2318 
2319 void netif_tx_wake_queue(struct netdev_queue *dev_queue)
2320 {
2321         if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
2322                 struct Qdisc *q;
2323 
2324                 rcu_read_lock();
2325                 q = rcu_dereference(dev_queue->qdisc);
2326                 __netif_schedule(q);
2327                 rcu_read_unlock();
2328         }
2329 }
2330 EXPORT_SYMBOL(netif_tx_wake_queue);
2331 
2332 void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason)
2333 {
2334         unsigned long flags;
2335 
2336         if (likely(atomic_read(&skb->users) == 1)) {
2337                 smp_rmb();
2338                 atomic_set(&skb->users, 0);
2339         } else if (likely(!atomic_dec_and_test(&skb->users))) {
2340                 return;
2341         }
2342         get_kfree_skb_cb(skb)->reason = reason;
2343         local_irq_save(flags);
2344         skb->next = __this_cpu_read(softnet_data.completion_queue);
2345         __this_cpu_write(softnet_data.completion_queue, skb);
2346         raise_softirq_irqoff(NET_TX_SOFTIRQ);
2347         local_irq_restore(flags);
2348 }
2349 EXPORT_SYMBOL(__dev_kfree_skb_irq);
2350 
2351 void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason)
2352 {
2353         if (in_irq() || irqs_disabled())
2354                 __dev_kfree_skb_irq(skb, reason);
2355         else
2356                 dev_kfree_skb(skb);
2357 }
2358 EXPORT_SYMBOL(__dev_kfree_skb_any);
2359 
2360 
2361 /**
2362  * netif_device_detach - mark device as removed
2363  * @dev: network device
2364  *
2365  * Mark device as removed from system and therefore no longer available.
2366  */
2367 void netif_device_detach(struct net_device *dev)
2368 {
2369         if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
2370             netif_running(dev)) {
2371                 netif_tx_stop_all_queues(dev);
2372         }
2373 }
2374 EXPORT_SYMBOL(netif_device_detach);
2375 
2376 /**
2377  * netif_device_attach - mark device as attached
2378  * @dev: network device
2379  *
2380  * Mark device as attached from system and restart if needed.
2381  */
2382 void netif_device_attach(struct net_device *dev)
2383 {
2384         if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
2385             netif_running(dev)) {
2386                 netif_tx_wake_all_queues(dev);
2387                 __netdev_watchdog_up(dev);
2388         }
2389 }
2390 EXPORT_SYMBOL(netif_device_attach);
2391 
2392 /*
2393  * Returns a Tx hash based on the given packet descriptor a Tx queues' number
2394  * to be used as a distribution range.
2395  */
2396 u16 __skb_tx_hash(const struct net_device *dev, struct sk_buff *skb,
2397                   unsigned int num_tx_queues)
2398 {
2399         u32 hash;
2400         u16 qoffset = 0;
2401         u16 qcount = num_tx_queues;
2402 
2403         if (skb_rx_queue_recorded(skb)) {
2404                 hash = skb_get_rx_queue(skb);
2405                 while (unlikely(hash >= num_tx_queues))
2406                         hash -= num_tx_queues;
2407                 return hash;
2408         }
2409 
2410         if (dev->num_tc) {
2411                 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
2412                 qoffset = dev->tc_to_txq[tc].offset;
2413                 qcount = dev->tc_to_txq[tc].count;
2414         }
2415 
2416         return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
2417 }
2418 EXPORT_SYMBOL(__skb_tx_hash);
2419 
2420 static void skb_warn_bad_offload(const struct sk_buff *skb)
2421 {
2422         static const netdev_features_t null_features = 0;
2423         struct net_device *dev = skb->dev;
2424         const char *name = "";
2425 
2426         if (!net_ratelimit())
2427                 return;
2428 
2429         if (dev) {
2430                 if (dev->dev.parent)
2431                         name = dev_driver_string(dev->dev.parent);
2432                 else
2433                         name = netdev_name(dev);
2434         }
2435         WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d "
2436              "gso_type=%d ip_summed=%d\n",
2437              name, dev ? &dev->features : &null_features,
2438              skb->sk ? &skb->sk->sk_route_caps : &null_features,
2439              skb->len, skb->data_len, skb_shinfo(skb)->gso_size,
2440              skb_shinfo(skb)->gso_type, skb->ip_summed);
2441 }
2442 
2443 /*
2444  * Invalidate hardware checksum when packet is to be mangled, and
2445  * complete checksum manually on outgoing path.
2446  */
2447 int skb_checksum_help(struct sk_buff *skb)
2448 {
2449         __wsum csum;
2450         int ret = 0, offset;
2451 
2452         if (skb->ip_summed == CHECKSUM_COMPLETE)
2453                 goto out_set_summed;
2454 
2455         if (unlikely(skb_shinfo(skb)->gso_size)) {
2456                 skb_warn_bad_offload(skb);
2457                 return -EINVAL;
2458         }
2459 
2460         /* Before computing a checksum, we should make sure no frag could
2461          * be modified by an external entity : checksum could be wrong.
2462          */
2463         if (skb_has_shared_frag(skb)) {
2464                 ret = __skb_linearize(skb);
2465                 if (ret)
2466                         goto out;
2467         }
2468 
2469         offset = skb_checksum_start_offset(skb);
2470         BUG_ON(offset >= skb_headlen(skb));
2471         csum = skb_checksum(skb, offset, skb->len - offset, 0);
2472 
2473         offset += skb->csum_offset;
2474         BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
2475 
2476         if (skb_cloned(skb) &&
2477             !skb_clone_writable(skb, offset + sizeof(__sum16))) {
2478                 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2479                 if (ret)
2480                         goto out;
2481         }
2482 
2483         *(__sum16 *)(skb->data + offset) = csum_fold(csum);
2484 out_set_summed:
2485         skb->ip_summed = CHECKSUM_NONE;
2486 out:
2487         return ret;
2488 }
2489 EXPORT_SYMBOL(skb_checksum_help);
2490 
2491 /* skb_csum_offload_check - Driver helper function to determine if a device
2492  * with limited checksum offload capabilities is able to offload the checksum
2493  * for a given packet.
2494  *
2495  * Arguments:
2496  *   skb - sk_buff for the packet in question
2497  *   spec - contains the description of what device can offload
2498  *   csum_encapped - returns true if the checksum being offloaded is
2499  *            encpasulated. That is it is checksum for the transport header
2500  *            in the inner headers.
2501  *   checksum_help - when set indicates that helper function should
2502  *            call skb_checksum_help if offload checks fail
2503  *
2504  * Returns:
2505  *   true: Packet has passed the checksum checks and should be offloadable to
2506  *         the device (a driver may still need to check for additional
2507  *         restrictions of its device)
2508  *   false: Checksum is not offloadable. If checksum_help was set then
2509  *         skb_checksum_help was called to resolve checksum for non-GSO
2510  *         packets and when IP protocol is not SCTP
2511  */
2512 bool __skb_csum_offload_chk(struct sk_buff *skb,
2513                             const struct skb_csum_offl_spec *spec,
2514                             bool *csum_encapped,
2515                             bool csum_help)
2516 {
2517         struct iphdr *iph;
2518         struct ipv6hdr *ipv6;
2519         void *nhdr;
2520         int protocol;
2521         u8 ip_proto;
2522 
2523         if (skb->protocol == htons(ETH_P_8021Q) ||
2524             skb->protocol == htons(ETH_P_8021AD)) {
2525                 if (!spec->vlan_okay)
2526                         goto need_help;
2527         }
2528 
2529         /* We check whether the checksum refers to a transport layer checksum in
2530          * the outermost header or an encapsulated transport layer checksum that
2531          * corresponds to the inner headers of the skb. If the checksum is for
2532          * something else in the packet we need help.
2533          */
2534         if (skb_checksum_start_offset(skb) == skb_transport_offset(skb)) {
2535                 /* Non-encapsulated checksum */
2536                 protocol = eproto_to_ipproto(vlan_get_protocol(skb));
2537                 nhdr = skb_network_header(skb);
2538                 *csum_encapped = false;
2539                 if (spec->no_not_encapped)
2540                         goto need_help;
2541         } else if (skb->encapsulation && spec->encap_okay &&
2542                    skb_checksum_start_offset(skb) ==
2543                    skb_inner_transport_offset(skb)) {
2544                 /* Encapsulated checksum */
2545                 *csum_encapped = true;
2546                 switch (skb->inner_protocol_type) {
2547                 case ENCAP_TYPE_ETHER:
2548                         protocol = eproto_to_ipproto(skb->inner_protocol);
2549                         break;
2550                 case ENCAP_TYPE_IPPROTO:
2551                         protocol = skb->inner_protocol;
2552                         break;
2553                 }
2554                 nhdr = skb_inner_network_header(skb);
2555         } else {
2556                 goto need_help;
2557         }
2558 
2559         switch (protocol) {
2560         case IPPROTO_IP:
2561                 if (!spec->ipv4_okay)
2562                         goto need_help;
2563                 iph = nhdr;
2564                 ip_proto = iph->protocol;
2565                 if (iph->ihl != 5 && !spec->ip_options_okay)
2566                         goto need_help;
2567                 break;
2568         case IPPROTO_IPV6:
2569                 if (!spec->ipv6_okay)
2570                         goto need_help;
2571                 if (spec->no_encapped_ipv6 && *csum_encapped)
2572                         goto need_help;
2573                 ipv6 = nhdr;
2574                 nhdr += sizeof(*ipv6);
2575                 ip_proto = ipv6->nexthdr;
2576                 break;
2577         default:
2578                 goto need_help;
2579         }
2580 
2581 ip_proto_again:
2582         switch (ip_proto) {
2583         case IPPROTO_TCP:
2584                 if (!spec->tcp_okay ||
2585                     skb->csum_offset != offsetof(struct tcphdr, check))
2586                         goto need_help;
2587                 break;
2588         case IPPROTO_UDP:
2589                 if (!spec->udp_okay ||
2590                     skb->csum_offset != offsetof(struct udphdr, check))
2591                         goto need_help;
2592                 break;
2593         case IPPROTO_SCTP:
2594                 if (!spec->sctp_okay ||
2595                     skb->csum_offset != offsetof(struct sctphdr, checksum))
2596                         goto cant_help;
2597                 break;
2598         case NEXTHDR_HOP:
2599         case NEXTHDR_ROUTING:
2600         case NEXTHDR_DEST: {
2601                 u8 *opthdr = nhdr;
2602 
2603                 if (protocol != IPPROTO_IPV6 || !spec->ext_hdrs_okay)
2604                         goto need_help;
2605 
2606                 ip_proto = opthdr[0];
2607                 nhdr += (opthdr[1] + 1) << 3;
2608 
2609                 goto ip_proto_again;
2610         }
2611         default:
2612                 goto need_help;
2613         }
2614 
2615         /* Passed the tests for offloading checksum */
2616         return true;
2617 
2618 need_help:
2619         if (csum_help && !skb_shinfo(skb)->gso_size)
2620                 skb_checksum_help(skb);
2621 cant_help:
2622         return false;
2623 }
2624 EXPORT_SYMBOL(__skb_csum_offload_chk);
2625 
2626 __be16 skb_network_protocol(struct sk_buff *skb, int *depth)
2627 {
2628         __be16 type = skb->protocol;
2629 
2630         /* Tunnel gso handlers can set protocol to ethernet. */
2631         if (type == htons(ETH_P_TEB)) {
2632                 struct ethhdr *eth;
2633 
2634                 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
2635                         return 0;
2636 
2637                 eth = (struct ethhdr *)skb_mac_header(skb);
2638                 type = eth->h_proto;
2639         }
2640 
2641         return __vlan_get_protocol(skb, type, depth);
2642 }
2643 
2644 /**
2645  *      skb_mac_gso_segment - mac layer segmentation handler.
2646  *      @skb: buffer to segment
2647  *      @features: features for the output path (see dev->features)
2648  */
2649 struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
2650                                     netdev_features_t features)
2651 {
2652         struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
2653         struct packet_offload *ptype;
2654         int vlan_depth = skb->mac_len;
2655         __be16 type = skb_network_protocol(skb, &vlan_depth);
2656 
2657         if (unlikely(!type))
2658                 return ERR_PTR(-EINVAL);
2659 
2660         __skb_pull(skb, vlan_depth);
2661 
2662         rcu_read_lock();
2663         list_for_each_entry_rcu(ptype, &offload_base, list) {
2664                 if (ptype->type == type && ptype->callbacks.gso_segment) {
2665                         segs = ptype->callbacks.gso_segment(skb, features);
2666                         break;
2667                 }
2668         }
2669         rcu_read_unlock();
2670 
2671         __skb_push(skb, skb->data - skb_mac_header(skb));
2672 
2673         return segs;
2674 }
2675 EXPORT_SYMBOL(skb_mac_gso_segment);
2676 
2677 
2678 /* openvswitch calls this on rx path, so we need a different check.
2679  */
2680 static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
2681 {
2682         if (tx_path)
2683                 return skb->ip_summed != CHECKSUM_PARTIAL;
2684         else
2685                 return skb->ip_summed == CHECKSUM_NONE;
2686 }
2687 
2688 /**
2689  *      __skb_gso_segment - Perform segmentation on skb.
2690  *      @skb: buffer to segment
2691  *      @features: features for the output path (see dev->features)
2692  *      @tx_path: whether it is called in TX path
2693  *
2694  *      This function segments the given skb and returns a list of segments.
2695  *
2696  *      It may return NULL if the skb requires no segmentation.  This is
2697  *      only possible when GSO is used for verifying header integrity.
2698  *
2699  *      Segmentation preserves SKB_SGO_CB_OFFSET bytes of previous skb cb.
2700  */
2701 struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
2702                                   netdev_features_t features, bool tx_path)
2703 {
2704         if (unlikely(skb_needs_check(skb, tx_path))) {
2705                 int err;
2706 
2707                 skb_warn_bad_offload(skb);
2708 
2709                 err = skb_cow_head(skb, 0);
2710                 if (err < 0)
2711                         return ERR_PTR(err);
2712         }
2713 
2714         BUILD_BUG_ON(SKB_SGO_CB_OFFSET +
2715                      sizeof(*SKB_GSO_CB(skb)) > sizeof(skb->cb));
2716 
2717         SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
2718         SKB_GSO_CB(skb)->encap_level = 0;
2719 
2720         skb_reset_mac_header(skb);
2721         skb_reset_mac_len(skb);
2722 
2723         return skb_mac_gso_segment(skb, features);
2724 }
2725 EXPORT_SYMBOL(__skb_gso_segment);
2726 
2727 /* Take action when hardware reception checksum errors are detected. */
2728 #ifdef CONFIG_BUG
2729 void netdev_rx_csum_fault(struct net_device *dev)
2730 {
2731         if (net_ratelimit()) {
2732                 pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
2733                 dump_stack();
2734         }
2735 }
2736 EXPORT_SYMBOL(netdev_rx_csum_fault);
2737 #endif
2738 
2739 /* Actually, we should eliminate this check as soon as we know, that:
2740  * 1. IOMMU is present and allows to map all the memory.
2741  * 2. No high memory really exists on this machine.
2742  */
2743 
2744 static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
2745 {
2746 #ifdef CONFIG_HIGHMEM
2747         int i;
2748         if (!(dev->features & NETIF_F_HIGHDMA)) {
2749                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2750                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2751                         if (PageHighMem(skb_frag_page(frag)))
2752                                 return 1;
2753                 }
2754         }
2755 
2756         if (PCI_DMA_BUS_IS_PHYS) {
2757                 struct device *pdev = dev->dev.parent;
2758 
2759                 if (!pdev)
2760                         return 0;
2761                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2762                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2763                         dma_addr_t addr = page_to_phys(skb_frag_page(frag));
2764                         if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
2765                                 return 1;
2766                 }
2767         }
2768 #endif
2769         return 0;
2770 }
2771 
2772 /* If MPLS offload request, verify we are testing hardware MPLS features
2773  * instead of standard features for the netdev.
2774  */
2775 #if IS_ENABLED(CONFIG_NET_MPLS_GSO)
2776 static netdev_features_t net_mpls_features(struct sk_buff *skb,
2777                                            netdev_features_t features,
2778                                            __be16 type)
2779 {
2780         if (eth_p_mpls(type))
2781                 features &= skb->dev->mpls_features;
2782 
2783         return features;
2784 }
2785 #else
2786 static netdev_features_t net_mpls_features(struct sk_buff *skb,
2787                                            netdev_features_t features,
2788                                            __be16 type)
2789 {
2790         return features;
2791 }
2792 #endif
2793 
2794 static netdev_features_t harmonize_features(struct sk_buff *skb,
2795         netdev_features_t features)
2796 {
2797         int tmp;
2798         __be16 type;
2799 
2800         type = skb_network_protocol(skb, &tmp);
2801         features = net_mpls_features(skb, features, type);
2802 
2803         if (skb->ip_summed != CHECKSUM_NONE &&
2804             !can_checksum_protocol(features, type)) {
2805                 features &= ~NETIF_F_CSUM_MASK;
2806         } else if (illegal_highdma(skb->dev, skb)) {
2807                 features &= ~NETIF_F_SG;
2808         }
2809 
2810         return features;
2811 }
2812 
2813 netdev_features_t passthru_features_check(struct sk_buff *skb,
2814                                           struct net_device *dev,
2815                                           netdev_features_t features)
2816 {
2817         return features;
2818 }
2819 EXPORT_SYMBOL(passthru_features_check);
2820 
2821 static netdev_features_t dflt_features_check(const struct sk_buff *skb,
2822                                              struct net_device *dev,
2823                                              netdev_features_t features)
2824 {
2825         return vlan_features_check(skb, features);
2826 }
2827 
2828 netdev_features_t netif_skb_features(struct sk_buff *skb)
2829 {
2830         struct net_device *dev = skb->dev;
2831         netdev_features_t features = dev->features;
2832         u16 gso_segs = skb_shinfo(skb)->gso_segs;
2833 
2834         if (gso_segs > dev->gso_max_segs || gso_segs < dev->gso_min_segs)
2835                 features &= ~NETIF_F_GSO_MASK;
2836 
2837         /* If encapsulation offload request, verify we are testing
2838          * hardware encapsulation features instead of standard
2839          * features for the netdev
2840          */
2841         if (skb->encapsulation)
2842                 features &= dev->hw_enc_features;
2843 
2844         if (skb_vlan_tagged(skb))
2845                 features = netdev_intersect_features(features,
2846                                                      dev->vlan_features |
2847                                                      NETIF_F_HW_VLAN_CTAG_TX |
2848                                                      NETIF_F_HW_VLAN_STAG_TX);
2849 
2850         if (dev->netdev_ops->ndo_features_check)
2851                 features &= dev->netdev_ops->ndo_features_check(skb, dev,
2852                                                                 features);
2853         else
2854                 features &= dflt_features_check(skb, dev, features);
2855 
2856         return harmonize_features(skb, features);
2857 }
2858 EXPORT_SYMBOL(netif_skb_features);
2859 
2860 static int xmit_one(struct sk_buff *skb, struct net_device *dev,
2861                     struct netdev_queue *txq, bool more)
2862 {
2863         unsigned int len;
2864         int rc;
2865 
2866         if (!list_empty(&ptype_all) || !list_empty(&dev->ptype_all))
2867                 dev_queue_xmit_nit(skb, dev);
2868 
2869         len = skb->len;
2870         trace_net_dev_start_xmit(skb, dev);
2871         rc = netdev_start_xmit(skb, dev, txq, more);
2872         trace_net_dev_xmit(skb, rc, dev, len);
2873 
2874         return rc;
2875 }
2876 
2877 struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
2878                                     struct netdev_queue *txq, int *ret)
2879 {
2880         struct sk_buff *skb = first;
2881         int rc = NETDEV_TX_OK;
2882 
2883         while (skb) {
2884                 struct sk_buff *next = skb->next;
2885 
2886                 skb->next = NULL;
2887                 rc = xmit_one(skb, dev, txq, next != NULL);
2888                 if (unlikely(!dev_xmit_complete(rc))) {
2889                         skb->next = next;
2890                         goto out;
2891                 }
2892 
2893                 skb = next;
2894                 if (netif_xmit_stopped(txq) && skb) {
2895                         rc = NETDEV_TX_BUSY;
2896                         break;
2897                 }
2898         }
2899 
2900 out:
2901         *ret = rc;
2902         return skb;
2903 }
2904 
2905 static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
2906                                           netdev_features_t features)
2907 {
2908         if (skb_vlan_tag_present(skb) &&
2909             !vlan_hw_offload_capable(features, skb->vlan_proto))
2910                 skb = __vlan_hwaccel_push_inside(skb);
2911         return skb;
2912 }
2913 
2914 static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev)
2915 {
2916         netdev_features_t features;
2917 
2918         if (skb->next)
2919                 return skb;
2920 
2921         features = netif_skb_features(skb);
2922         skb = validate_xmit_vlan(skb, features);
2923         if (unlikely(!skb))
2924                 goto out_null;
2925 
2926         if (netif_needs_gso(skb, features)) {
2927                 struct sk_buff *segs;
2928 
2929                 segs = skb_gso_segment(skb, features);
2930                 if (IS_ERR(segs)) {
2931                         goto out_kfree_skb;
2932                 } else if (segs) {
2933                         consume_skb(skb);
2934                         skb = segs;
2935                 }
2936         } else {
2937                 if (skb_needs_linearize(skb, features) &&
2938                     __skb_linearize(skb))
2939                         goto out_kfree_skb;
2940 
2941                 /* If packet is not checksummed and device does not
2942                  * support checksumming for this protocol, complete
2943                  * checksumming here.
2944                  */
2945                 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2946                         if (skb->encapsulation)
2947                                 skb_set_inner_transport_header(skb,
2948                                                                skb_checksum_start_offset(skb));
2949                         else
2950                                 skb_set_transport_header(skb,
2951                                                          skb_checksum_start_offset(skb));
2952                         if (!(features & NETIF_F_CSUM_MASK) &&
2953                             skb_checksum_help(skb))
2954                                 goto out_kfree_skb;
2955                 }
2956         }
2957 
2958         return skb;
2959 
2960 out_kfree_skb:
2961         kfree_skb(skb);
2962 out_null:
2963         return NULL;
2964 }
2965 
2966 struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev)
2967 {
2968         struct sk_buff *next, *head = NULL, *tail;
2969 
2970         for (; skb != NULL; skb = next) {
2971                 next = skb->next;
2972                 skb->next = NULL;
2973 
2974                 /* in case skb wont be segmented, point to itself */
2975                 skb->prev = skb;
2976 
2977                 skb = validate_xmit_skb(skb, dev);
2978                 if (!skb)
2979                         continue;
2980 
2981                 if (!head)
2982                         head = skb;
2983                 else
2984                         tail->next = skb;
2985                 /* If skb was segmented, skb->prev points to
2986                  * the last segment. If not, it still contains skb.
2987                  */
2988                 tail = skb->prev;
2989         }
2990         return head;
2991 }
2992 
2993 static void qdisc_pkt_len_init(struct sk_buff *skb)
2994 {
2995         const struct skb_shared_info *shinfo = skb_shinfo(skb);
2996 
2997         qdisc_skb_cb(skb)->pkt_len = skb->len;
2998 
2999         /* To get more precise estimation of bytes sent on wire,
3000          * we add to pkt_len the headers size of all segments
3001          */
3002         if (shinfo->gso_size)  {
3003                 unsigned int hdr_len;
3004                 u16 gso_segs = shinfo->gso_segs;
3005 
3006                 /* mac layer + network layer */
3007                 hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
3008 
3009                 /* + transport layer */
3010                 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
3011                         hdr_len += tcp_hdrlen(skb);
3012                 else
3013                         hdr_len += sizeof(struct udphdr);
3014 
3015                 if (shinfo->gso_type & SKB_GSO_DODGY)
3016                         gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
3017                                                 shinfo->gso_size);
3018 
3019                 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
3020         }
3021 }
3022 
3023 static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
3024                                  struct net_device *dev,
3025                                  struct netdev_queue *txq)
3026 {
3027         spinlock_t *root_lock = qdisc_lock(q);
3028         bool contended;
3029         int rc;
3030 
3031         qdisc_calculate_pkt_len(skb, q);
3032         /*
3033          * Heuristic to force contended enqueues to serialize on a
3034          * separate lock before trying to get qdisc main lock.
3035          * This permits __QDISC___STATE_RUNNING owner to get the lock more
3036          * often and dequeue packets faster.
3037          */
3038         contended = qdisc_is_running(q);
3039         if (unlikely(contended))
3040                 spin_lock(&q->busylock);
3041 
3042         spin_lock(root_lock);
3043         if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
3044                 kfree_skb(skb);
3045                 rc = NET_XMIT_DROP;
3046         } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
3047                    qdisc_run_begin(q)) {
3048                 /*
3049                  * This is a work-conserving queue; there are no old skbs
3050                  * waiting to be sent out; and the qdisc is not running -
3051                  * xmit the skb directly.
3052                  */
3053 
3054                 qdisc_bstats_update(q, skb);
3055 
3056                 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
3057                         if (unlikely(contended)) {
3058                                 spin_unlock(&q->busylock);
3059                                 contended = false;
3060                         }
3061                         __qdisc_run(q);
3062                 } else
3063                         qdisc_run_end(q);
3064 
3065                 rc = NET_XMIT_SUCCESS;
3066         } else {
3067                 rc = q->enqueue(skb, q) & NET_XMIT_MASK;
3068                 if (qdisc_run_begin(q)) {
3069                         if (unlikely(contended)) {
3070                                 spin_unlock(&q->busylock);
3071                                 contended = false;
3072                         }
3073                         __qdisc_run(q);
3074                 }
3075         }
3076         spin_unlock(root_lock);
3077         if (unlikely(contended))
3078                 spin_unlock(&q->busylock);
3079         return rc;
3080 }
3081 
3082 #if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
3083 static void skb_update_prio(struct sk_buff *skb)
3084 {
3085         struct netprio_map *map = rcu_dereference_bh(skb->dev->priomap);
3086 
3087         if (!skb->priority && skb->sk && map) {
3088                 unsigned int prioidx =
3089                         sock_cgroup_prioidx(&skb->sk->sk_cgrp_data);
3090 
3091                 if (prioidx < map->priomap_len)
3092                         skb->priority = map->priomap[prioidx];
3093         }
3094 }
3095 #else
3096 #define skb_update_prio(skb)
3097 #endif
3098 
3099 DEFINE_PER_CPU(int, xmit_recursion);
3100 EXPORT_SYMBOL(xmit_recursion);
3101 
3102 #define RECURSION_LIMIT 10
3103 
3104 /**
3105  *      dev_loopback_xmit - loop back @skb
3106  *      @net: network namespace this loopback is happening in
3107  *      @sk:  sk needed to be a netfilter okfn
3108  *      @skb: buffer to transmit
3109  */
3110 int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb)
3111 {
3112         skb_reset_mac_header(skb);
3113         __skb_pull(skb, skb_network_offset(skb));
3114         skb->pkt_type = PACKET_LOOPBACK;
3115         skb->ip_summed = CHECKSUM_UNNECESSARY;
3116         WARN_ON(!skb_dst(skb));
3117         skb_dst_force(skb);
3118         netif_rx_ni(skb);
3119         return 0;
3120 }
3121 EXPORT_SYMBOL(dev_loopback_xmit);
3122 
3123 #ifdef CONFIG_NET_EGRESS
3124 static struct sk_buff *
3125 sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
3126 {
3127         struct tcf_proto *cl = rcu_dereference_bh(dev->egress_cl_list);
3128         struct tcf_result cl_res;
3129 
3130         if (!cl)
3131                 return skb;
3132 
3133         /* skb->tc_verd and qdisc_skb_cb(skb)->pkt_len were already set
3134          * earlier by the caller.
3135          */
3136         qdisc_bstats_cpu_update(cl->q, skb);
3137 
3138         switch (tc_classify(skb, cl, &cl_res, false)) {
3139         case TC_ACT_OK:
3140         case TC_ACT_RECLASSIFY:
3141                 skb->tc_index = TC_H_MIN(cl_res.classid);
3142                 break;
3143         case TC_ACT_SHOT:
3144                 qdisc_qstats_cpu_drop(cl->q);
3145                 *ret = NET_XMIT_DROP;
3146                 goto drop;
3147         case TC_ACT_STOLEN:
3148         case TC_ACT_QUEUED:
3149                 *ret = NET_XMIT_SUCCESS;
3150 drop:
3151                 kfree_skb(skb);
3152                 return NULL;
3153         case TC_ACT_REDIRECT:
3154                 /* No need to push/pop skb's mac_header here on egress! */
3155                 skb_do_redirect(skb);
3156                 *ret = NET_XMIT_SUCCESS;
3157                 return NULL;
3158         default:
3159                 break;
3160         }
3161 
3162         return skb;
3163 }
3164 #endif /* CONFIG_NET_EGRESS */
3165 
3166 static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb)
3167 {
3168 #ifdef CONFIG_XPS
3169         struct xps_dev_maps *dev_maps;
3170         struct xps_map *map;
3171         int queue_index = -1;
3172 
3173         rcu_read_lock();
3174         dev_maps = rcu_dereference(dev->xps_maps);
3175         if (dev_maps) {
3176                 map = rcu_dereference(
3177                     dev_maps->cpu_map[skb->sender_cpu - 1]);
3178                 if (map) {
3179                         if (map->len == 1)
3180                                 queue_index = map->queues[0];
3181                         else
3182                                 queue_index = map->queues[reciprocal_scale(skb_get_hash(skb),
3183                                                                            map->len)];
3184                         if (unlikely(queue_index >= dev->real_num_tx_queues))
3185                                 queue_index = -1;
3186                 }
3187         }
3188         rcu_read_unlock();
3189 
3190         return queue_index;
3191 #else
3192         return -1;
3193 #endif
3194 }
3195 
3196 static u16 __netdev_pick_tx(struct net_device *dev, struct sk_buff *skb)
3197 {
3198         struct sock *sk = skb->sk;
3199         int queue_index = sk_tx_queue_get(sk);
3200 
3201         if (queue_index < 0 || skb->ooo_okay ||
3202             queue_index >= dev->real_num_tx_queues) {
3203                 int new_index = get_xps_queue(dev, skb);
3204                 if (new_index < 0)
3205                         new_index = skb_tx_hash(dev, skb);
3206 
3207                 if (queue_index != new_index && sk &&
3208                     sk_fullsock(sk) &&
3209                     rcu_access_pointer(sk->sk_dst_cache))
3210                         sk_tx_queue_set(sk, new_index);
3211 
3212                 queue_index = new_index;
3213         }
3214 
3215         return queue_index;
3216 }
3217 
3218 struct netdev_queue *netdev_pick_tx(struct net_device *dev,
3219                                     struct sk_buff *skb,
3220                                     void *accel_priv)
3221 {
3222         int queue_index = 0;
3223 
3224 #ifdef CONFIG_XPS
3225         u32 sender_cpu = skb->sender_cpu - 1;
3226 
3227         if (sender_cpu >= (u32)NR_CPUS)
3228                 skb->sender_cpu = raw_smp_processor_id() + 1;
3229 #endif
3230 
3231         if (dev->real_num_tx_queues != 1) {
3232                 const struct net_device_ops *ops = dev->netdev_ops;
3233                 if (ops->ndo_select_queue)
3234                         queue_index = ops->ndo_select_queue(dev, skb, accel_priv,
3235                                                             __netdev_pick_tx);
3236                 else
3237                         queue_index = __netdev_pick_tx(dev, skb);
3238 
3239                 if (!accel_priv)
3240                         queue_index = netdev_cap_txqueue(dev, queue_index);
3241         }
3242 
3243         skb_set_queue_mapping(skb, queue_index);
3244         return netdev_get_tx_queue(dev, queue_index);
3245 }
3246 
3247 /**
3248  *      __dev_queue_xmit - transmit a buffer
3249  *      @skb: buffer to transmit
3250  *      @accel_priv: private data used for L2 forwarding offload
3251  *
3252  *      Queue a buffer for transmission to a network device. The caller must
3253  *      have set the device and priority and built the buffer before calling
3254  *      this function. The function can be called from an interrupt.
3255  *
3256  *      A negative errno code is returned on a failure. A success does not
3257  *      guarantee the frame will be transmitted as it may be dropped due
3258  *      to congestion or traffic shaping.
3259  *
3260  * -----------------------------------------------------------------------------------
3261  *      I notice this method can also return errors from the queue disciplines,
3262  *      including NET_XMIT_DROP, which is a positive value.  So, errors can also
3263  *      be positive.
3264  *
3265  *      Regardless of the return value, the skb is consumed, so it is currently
3266  *      difficult to retry a send to this method.  (You can bump the ref count
3267  *      before sending to hold a reference for retry if you are careful.)
3268  *
3269  *      When calling this method, interrupts MUST be enabled.  This is because
3270  *      the BH enable code must have IRQs enabled so that it will not deadlock.
3271  *          --BLG
3272  */
3273 static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv)
3274 {
3275         struct net_device *dev = skb->dev;
3276         struct netdev_queue *txq;
3277         struct Qdisc *q;
3278         int rc = -ENOMEM;
3279 
3280         skb_reset_mac_header(skb);
3281 
3282         if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
3283                 __skb_tstamp_tx(skb, NULL, skb->sk, SCM_TSTAMP_SCHED);
3284 
3285         /* Disable soft irqs for various locks below. Also
3286          * stops preemption for RCU.
3287          */
3288         rcu_read_lock_bh();
3289 
3290         skb_update_prio(skb);
3291 
3292         qdisc_pkt_len_init(skb);
3293 #ifdef CONFIG_NET_CLS_ACT
3294         skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS);
3295 # ifdef CONFIG_NET_EGRESS
3296         if (static_key_false(&egress_needed)) {
3297                 skb = sch_handle_egress(skb, &rc, dev);
3298                 if (!skb)
3299                         goto out;
3300         }
3301 # endif
3302 #endif
3303         /* If device/qdisc don't need skb->dst, release it right now while
3304          * its hot in this cpu cache.
3305          */
3306         if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
3307                 skb_dst_drop(skb);
3308         else
3309                 skb_dst_force(skb);
3310 
3311 #ifdef CONFIG_NET_SWITCHDEV
3312         /* Don't forward if offload device already forwarded */
3313         if (skb->offload_fwd_mark &&
3314             skb->offload_fwd_mark == dev->offload_fwd_mark) {
3315                 consume_skb(skb);
3316                 rc = NET_XMIT_SUCCESS;
3317                 goto out;
3318         }
3319 #endif
3320 
3321         txq = netdev_pick_tx(dev, skb, accel_priv);
3322         q = rcu_dereference_bh(txq->qdisc);
3323 
3324         trace_net_dev_queue(skb);
3325         if (q->enqueue) {
3326                 rc = __dev_xmit_skb(skb, q, dev, txq);
3327                 goto out;
3328         }
3329 
3330         /* The device has no queue. Common case for software devices:
3331            loopback, all the sorts of tunnels...
3332 
3333            Really, it is unlikely that netif_tx_lock protection is necessary
3334            here.  (f.e. loopback and IP tunnels are clean ignoring statistics
3335            counters.)
3336            However, it is possible, that they rely on protection
3337            made by us here.
3338 
3339            Check this and shot the lock. It is not prone from deadlocks.
3340            Either shot noqueue qdisc, it is even simpler 8)
3341          */
3342         if (dev->flags & IFF_UP) {
3343                 int cpu = smp_processor_id(); /* ok because BHs are off */
3344 
3345                 if (txq->xmit_lock_owner != cpu) {
3346 
3347                         if (__this_cpu_read(xmit_recursion) > RECURSION_LIMIT)
3348                                 goto recursion_alert;
3349 
3350                         skb = validate_xmit_skb(skb, dev);
3351                         if (!skb)
3352                                 goto drop;
3353 
3354                         HARD_TX_LOCK(dev, txq, cpu);
3355 
3356                         if (!netif_xmit_stopped(txq)) {
3357                                 __this_cpu_inc(xmit_recursion);
3358                                 skb = dev_hard_start_xmit(skb, dev, txq, &rc);
3359                                 __this_cpu_dec(xmit_recursion);
3360                                 if (dev_xmit_complete(rc)) {
3361                                         HARD_TX_UNLOCK(dev, txq);
3362                                         goto out;
3363                                 }
3364                         }
3365                         HARD_TX_UNLOCK(dev, txq);
3366                         net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
3367                                              dev->name);
3368                 } else {
3369                         /* Recursion is detected! It is possible,
3370                          * unfortunately
3371                          */
3372 recursion_alert:
3373                         net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
3374                                              dev->name);
3375                 }
3376         }
3377 
3378         rc = -ENETDOWN;
3379 drop:
3380         rcu_read_unlock_bh();
3381 
3382         atomic_long_inc(&dev->tx_dropped);
3383         kfree_skb_list(skb);
3384         return rc;
3385 out:
3386         rcu_read_unlock_bh();
3387         return rc;
3388 }
3389 
3390 int dev_queue_xmit(struct sk_buff *skb)
3391 {
3392         return __dev_queue_xmit(skb, NULL);
3393 }
3394 EXPORT_SYMBOL(dev_queue_xmit);
3395 
3396 int dev_queue_xmit_accel(struct sk_buff *skb, void *accel_priv)
3397 {
3398         return __dev_queue_xmit(skb, accel_priv);
3399 }
3400 EXPORT_SYMBOL(dev_queue_xmit_accel);
3401 
3402 
3403 /*=======================================================================
3404                         Receiver routines
3405   =======================================================================*/
3406 
3407 int netdev_max_backlog __read_mostly = 1000;
3408 EXPORT_SYMBOL(netdev_max_backlog);
3409 
3410 int netdev_tstamp_prequeue __read_mostly = 1;
3411 int netdev_budget __read_mostly = 300;
3412 int weight_p __read_mostly = 64;            /* old backlog weight */
3413 
3414 /* Called with irq disabled */
3415 static inline void ____napi_schedule(struct softnet_data *sd,
3416                                      struct napi_struct *napi)
3417 {
3418         list_add_tail(&napi->poll_list, &sd->poll_list);
3419         __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3420 }
3421 
3422 #ifdef CONFIG_RPS
3423 
3424 /* One global table that all flow-based protocols share. */
3425 struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
3426 EXPORT_SYMBOL(rps_sock_flow_table);
3427 u32 rps_cpu_mask __read_mostly;
3428 EXPORT_SYMBOL(rps_cpu_mask);
3429 
3430 struct static_key rps_needed __read_mostly;
3431 
3432 static struct rps_dev_flow *
3433 set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3434             struct rps_dev_flow *rflow, u16 next_cpu)
3435 {
3436         if (next_cpu < nr_cpu_ids) {
3437 #ifdef CONFIG_RFS_ACCEL
3438                 struct netdev_rx_queue *rxqueue;
3439                 struct rps_dev_flow_table *flow_table;
3440                 struct rps_dev_flow *old_rflow;
3441                 u32 flow_id;
3442                 u16 rxq_index;
3443                 int rc;
3444 
3445                 /* Should we steer this flow to a different hardware queue? */
3446                 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
3447                     !(dev->features & NETIF_F_NTUPLE))
3448                         goto out;
3449                 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
3450                 if (rxq_index == skb_get_rx_queue(skb))
3451                         goto out;
3452 
3453                 rxqueue = dev->_rx + rxq_index;
3454                 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3455                 if (!flow_table)
3456                         goto out;
3457                 flow_id = skb_get_hash(skb) & flow_table->mask;
3458                 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
3459                                                         rxq_index, flow_id);
3460                 if (rc < 0)
3461                         goto out;
3462                 old_rflow = rflow;
3463                 rflow = &flow_table->flows[flow_id];
3464                 rflow->filter = rc;
3465                 if (old_rflow->filter == rflow->filter)
3466                         old_rflow->filter = RPS_NO_FILTER;
3467         out:
3468 #endif
3469                 rflow->last_qtail =
3470                         per_cpu(softnet_data, next_cpu).input_queue_head;
3471         }
3472 
3473         rflow->cpu = next_cpu;
3474         return rflow;
3475 }
3476 
3477 /*
3478  * get_rps_cpu is called from netif_receive_skb and returns the target
3479  * CPU from the RPS map of the receiving queue for a given skb.
3480  * rcu_read_lock must be held on entry.
3481  */
3482 static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3483                        struct rps_dev_flow **rflowp)
3484 {
3485         const struct rps_sock_flow_table *sock_flow_table;
3486         struct netdev_rx_queue *rxqueue = dev->_rx;
3487         struct rps_dev_flow_table *flow_table;
3488         struct rps_map *map;
3489         int cpu = -1;
3490         u32 tcpu;
3491         u32 hash;
3492 
3493         if (skb_rx_queue_recorded(skb)) {
3494                 u16 index = skb_get_rx_queue(skb);
3495 
3496                 if (unlikely(index >= dev->real_num_rx_queues)) {
3497                         WARN_ONCE(dev->real_num_rx_queues > 1,
3498                                   "%s received packet on queue %u, but number "
3499                                   "of RX queues is %u\n",
3500                                   dev->name, index, dev->real_num_rx_queues);
3501                         goto done;
3502                 }
3503                 rxqueue += index;
3504         }
3505 
3506         /* Avoid computing hash if RFS/RPS is not active for this rxqueue */
3507 
3508         flow_table = rcu_dereference(rxqueue->rps_flow_table);
3509         map = rcu_dereference(rxqueue->rps_map);
3510         if (!flow_table && !map)
3511                 goto done;
3512 
3513         skb_reset_network_header(skb);
3514         hash = skb_get_hash(skb);
3515         if (!hash)
3516                 goto done;
3517 
3518         sock_flow_table = rcu_dereference(rps_sock_flow_table);
3519         if (flow_table && sock_flow_table) {
3520                 struct rps_dev_flow *rflow;
3521                 u32 next_cpu;
3522                 u32 ident;
3523 
3524                 /* First check into global flow table if there is a match */
3525                 ident = sock_flow_table->ents[hash & sock_flow_table->mask];
3526                 if ((ident ^ hash) & ~rps_cpu_mask)
3527                         goto try_rps;
3528 
3529                 next_cpu = ident & rps_cpu_mask;
3530 
3531                 /* OK, now we know there is a match,
3532                  * we can look at the local (per receive queue) flow table
3533                  */
3534                 rflow = &flow_table->flows[hash & flow_table->mask];
3535                 tcpu = rflow->cpu;
3536 
3537                 /*
3538                  * If the desired CPU (where last recvmsg was done) is
3539                  * different from current CPU (one in the rx-queue flow
3540                  * table entry), switch if one of the following holds:
3541                  *   - Current CPU is unset (>= nr_cpu_ids).
3542                  *   - Current CPU is offline.
3543                  *   - The current CPU's queue tail has advanced beyond the
3544                  *     last packet that was enqueued using this table entry.
3545                  *     This guarantees that all previous packets for the flow
3546                  *     have been dequeued, thus preserving in order delivery.
3547                  */
3548                 if (unlikely(tcpu != next_cpu) &&
3549                     (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
3550                      ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
3551                       rflow->last_qtail)) >= 0)) {
3552                         tcpu = next_cpu;
3553                         rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
3554                 }
3555 
3556                 if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
3557                         *rflowp = rflow;
3558                         cpu = tcpu;
3559                         goto done;
3560                 }
3561         }
3562 
3563 try_rps:
3564 
3565         if (map) {
3566                 tcpu = map->cpus[reciprocal_scale(hash, map->len)];
3567                 if (cpu_online(tcpu)) {
3568                         cpu = tcpu;
3569                         goto done;
3570                 }
3571         }
3572 
3573 done:
3574         return cpu;
3575 }
3576 
3577 #ifdef CONFIG_RFS_ACCEL
3578 
3579 /**
3580  * rps_may_expire_flow - check whether an RFS hardware filter may be removed
3581  * @dev: Device on which the filter was set
3582  * @rxq_index: RX queue index
3583  * @flow_id: Flow ID passed to ndo_rx_flow_steer()
3584  * @filter_id: Filter ID returned by ndo_rx_flow_steer()
3585  *
3586  * Drivers that implement ndo_rx_flow_steer() should periodically call
3587  * this function for each installed filter and remove the filters for
3588  * which it returns %true.
3589  */
3590 bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
3591                          u32 flow_id, u16 filter_id)
3592 {
3593         struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
3594         struct rps_dev_flow_table *flow_table;
3595         struct rps_dev_flow *rflow;
3596         bool expire = true;
3597         unsigned int cpu;
3598 
3599         rcu_read_lock();
3600         flow_table = rcu_dereference(rxqueue->rps_flow_table);
3601         if (flow_table && flow_id <= flow_table->mask) {
3602                 rflow = &flow_table->flows[flow_id];
3603                 cpu = ACCESS_ONCE(rflow->cpu);
3604                 if (rflow->filter == filter_id && cpu < nr_cpu_ids &&
3605                     ((int)(per_cpu(softnet_data, cpu).input_queue_head -
3606                            rflow->last_qtail) <
3607                      (int)(10 * flow_table->mask)))
3608                         expire = false;
3609         }
3610         rcu_read_unlock();
3611         return expire;
3612 }
3613 EXPORT_SYMBOL(rps_may_expire_flow);
3614 
3615 #endif /* CONFIG_RFS_ACCEL */
3616 
3617 /* Called from hardirq (IPI) context */
3618 static void rps_trigger_softirq(void *data)
3619 {
3620         struct softnet_data *sd = data;
3621 
3622         ____napi_schedule(sd, &sd->backlog);
3623         sd->received_rps++;
3624 }
3625 
3626 #endif /* CONFIG_RPS */
3627 
3628 /*
3629  * Check if this softnet_data structure is another cpu one
3630  * If yes, queue it to our IPI list and return 1
3631  * If no, return 0
3632  */
3633 static int rps_ipi_queued(struct softnet_data *sd)
3634 {
3635 #ifdef CONFIG_RPS
3636         struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
3637 
3638         if (sd != mysd) {
3639                 sd->rps_ipi_next = mysd->rps_ipi_list;
3640                 mysd->rps_ipi_list = sd;
3641 
3642                 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3643                 return 1;
3644         }
3645 #endif /* CONFIG_RPS */
3646         return 0;
3647 }
3648 
3649 #ifdef CONFIG_NET_FLOW_LIMIT
3650 int netdev_flow_limit_table_len __read_mostly = (1 << 12);
3651 #endif
3652 
3653 static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
3654 {
3655 #ifdef CONFIG_NET_FLOW_LIMIT
3656         struct sd_flow_limit *fl;
3657         struct softnet_data *sd;
3658         unsigned int old_flow, new_flow;
3659 
3660         if (qlen < (netdev_max_backlog >> 1))
3661                 return false;
3662 
3663         sd = this_cpu_ptr(&softnet_data);
3664 
3665         rcu_read_lock();
3666         fl = rcu_dereference(sd->flow_limit);
3667         if (fl) {
3668                 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
3669                 old_flow = fl->history[fl->history_head];
3670                 fl->history[fl->history_head] = new_flow;
3671 
3672                 fl->history_head++;
3673                 fl->history_head &= FLOW_LIMIT_HISTORY - 1;
3674 
3675                 if (likely(fl->buckets[old_flow]))
3676                         fl->buckets[old_flow]--;
3677 
3678                 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
3679                         fl->count++;
3680                         rcu_read_unlock();
3681                         return true;
3682                 }
3683         }
3684         rcu_read_unlock();
3685 #endif
3686         return false;
3687 }
3688 
3689 /*
3690  * enqueue_to_backlog is called to queue an skb to a per CPU backlog
3691  * queue (may be a remote CPU queue).
3692  */
3693 static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
3694                               unsigned int *qtail)
3695 {
3696         struct softnet_data *sd;
3697         unsigned long flags;
3698         unsigned int qlen;
3699 
3700         sd = &per_cpu(softnet_data, cpu);
3701 
3702         local_irq_save(flags);
3703 
3704         rps_lock(sd);
3705         if (!netif_running(skb->dev))
3706                 goto drop;
3707         qlen = skb_queue_len(&sd->input_pkt_queue);
3708         if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) {
3709                 if (qlen) {
3710 enqueue:
3711                         __skb_queue_tail(&sd->input_pkt_queue, skb);
3712                         input_queue_tail_incr_save(sd, qtail);
3713                         rps_unlock(sd);
3714                         local_irq_restore(flags);
3715                         return NET_RX_SUCCESS;
3716                 }
3717 
3718                 /* Schedule NAPI for backlog device
3719                  * We can use non atomic operation since we own the queue lock
3720                  */
3721                 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
3722                         if (!rps_ipi_queued(sd))
3723                                 ____napi_schedule(sd, &sd->backlog);
3724                 }
3725                 goto enqueue;
3726         }
3727 
3728 drop:
3729         sd->dropped++;
3730         rps_unlock(sd);
3731 
3732         local_irq_restore(flags);
3733 
3734         atomic_long_inc(&skb->dev->rx_dropped);
3735         kfree_skb(skb);
3736         return NET_RX_DROP;
3737 }
3738 
3739 static int netif_rx_internal(struct sk_buff *skb)
3740 {
3741         int ret;
3742 
3743         net_timestamp_check(netdev_tstamp_prequeue, skb);
3744 
3745         trace_netif_rx(skb);
3746 #ifdef CONFIG_RPS
3747         if (static_key_false(&rps_needed)) {
3748                 struct rps_dev_flow voidflow, *rflow = &voidflow;
3749                 int cpu;
3750 
3751                 preempt_disable();
3752                 rcu_read_lock();
3753 
3754                 cpu = get_rps_cpu(skb->dev, skb, &rflow);
3755                 if (cpu < 0)
3756                         cpu = smp_processor_id();
3757 
3758                 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3759 
3760                 rcu_read_unlock();
3761                 preempt_enable();
3762         } else
3763 #endif
3764         {
3765                 unsigned int qtail;
3766                 ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
3767                 put_cpu();
3768         }
3769         return ret;
3770 }
3771 
3772 /**
3773  *      netif_rx        -       post buffer to the network code
3774  *      @skb: buffer to post
3775  *
3776  *      This function receives a packet from a device driver and queues it for
3777  *      the upper (protocol) levels to process.  It always succeeds. The buffer
3778  *      may be dropped during processing for congestion control or by the
3779  *      protocol layers.
3780  *
3781  *      return values:
3782  *      NET_RX_SUCCESS  (no congestion)
3783  *      NET_RX_DROP     (packet was dropped)
3784  *
3785  */
3786 
3787 int netif_rx(struct sk_buff *skb)
3788 {
3789         trace_netif_rx_entry(skb);
3790 
3791         return netif_rx_internal(skb);
3792 }
3793 EXPORT_SYMBOL(netif_rx);
3794 
3795 int netif_rx_ni(struct sk_buff *skb)
3796 {
3797         int err;
3798 
3799         trace_netif_rx_ni_entry(skb);
3800 
3801         preempt_disable();
3802         err = netif_rx_internal(skb);
3803         if (local_softirq_pending())
3804                 do_softirq();
3805         preempt_enable();
3806 
3807         return err;
3808 }
3809 EXPORT_SYMBOL(netif_rx_ni);
3810 
3811 static void net_tx_action(struct softirq_action *h)
3812 {
3813         struct softnet_data *sd = this_cpu_ptr(&softnet_data);
3814 
3815         if (sd->completion_queue) {
3816                 struct sk_buff *clist;
3817 
3818                 local_irq_disable();
3819                 clist = sd->completion_queue;
3820                 sd->completion_queue = NULL;
3821                 local_irq_enable();
3822 
3823                 while (clist) {
3824                         struct sk_buff *skb = clist;
3825                         clist = clist->next;
3826 
3827                         WARN_ON(atomic_read(&skb->users));
3828                         if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED))
3829                                 trace_consume_skb(skb);
3830                         else
3831                                 trace_kfree_skb(skb, net_tx_action);
3832                         __kfree_skb(skb);
3833                 }
3834         }
3835 
3836         if (sd->output_queue) {
3837                 struct Qdisc *head;
3838 
3839                 local_irq_disable();
3840                 head = sd->output_queue;
3841                 sd->output_queue = NULL;
3842                 sd->output_queue_tailp = &sd->output_queue;
3843                 local_irq_enable();
3844 
3845                 while (head) {
3846                         struct Qdisc *q = head;
3847                         spinlock_t *root_lock;
3848 
3849                         head = head->next_sched;
3850 
3851                         root_lock = qdisc_lock(q);
3852                         if (spin_trylock(root_lock)) {
3853                                 smp_mb__before_atomic();
3854                                 clear_bit(__QDISC_STATE_SCHED,
3855                                           &q->state);
3856                                 qdisc_run(q);
3857                                 spin_unlock(root_lock);
3858                         } else {
3859                                 if (!test_bit(__QDISC_STATE_DEACTIVATED,
3860                                               &q->state)) {
3861                                         __netif_reschedule(q);
3862                                 } else {
3863                                         smp_mb__before_atomic();
3864                                         clear_bit(__QDISC_STATE_SCHED,
3865                                                   &q->state);
3866                                 }
3867                         }
3868                 }
3869         }
3870 }
3871 
3872 #if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \
3873     (defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE))
3874 /* This hook is defined here for ATM LANE */
3875 int (*br_fdb_test_addr_hook)(struct net_device *dev,
3876                              unsigned char *addr) __read_mostly;
3877 EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
3878 #endif
3879 
3880 static inline struct sk_buff *
3881 sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
3882                    struct net_device *orig_dev)
3883 {
3884 #ifdef CONFIG_NET_CLS_ACT
3885         struct tcf_proto *cl = rcu_dereference_bh(skb->dev->ingress_cl_list);
3886         struct tcf_result cl_res;
3887 
3888         /* If there's at least one ingress present somewhere (so
3889          * we get here via enabled static key), remaining devices
3890          * that are not configured with an ingress qdisc will bail
3891          * out here.
3892          */
3893         if (!cl)
3894                 return skb;
3895         if (*pt_prev) {
3896                 *ret = deliver_skb(skb, *pt_prev, orig_dev);
3897                 *pt_prev = NULL;
3898         }
3899 
3900         qdisc_skb_cb(skb)->pkt_len = skb->len;
3901         skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS);
3902         qdisc_bstats_cpu_update(cl->q, skb);
3903 
3904         switch (tc_classify(skb, cl, &cl_res, false)) {
3905         case TC_ACT_OK:
3906         case TC_ACT_RECLASSIFY:
3907                 skb->tc_index = TC_H_MIN(cl_res.classid);
3908                 break;
3909         case TC_ACT_SHOT:
3910                 qdisc_qstats_cpu_drop(cl->q);
3911         case TC_ACT_STOLEN:
3912         case TC_ACT_QUEUED:
3913                 kfree_skb(skb);
3914                 return NULL;
3915         case TC_ACT_REDIRECT:
3916                 /* skb_mac_header check was done by cls/act_bpf, so
3917                  * we can safely push the L2 header back before
3918                  * redirecting to another netdev
3919                  */
3920                 __skb_push(skb, skb->mac_len);
3921                 skb_do_redirect(skb);
3922                 return NULL;
3923         default:
3924                 break;
3925         }
3926 #endif /* CONFIG_NET_CLS_ACT */
3927         return skb;
3928 }
3929 
3930 /**
3931  *      netdev_rx_handler_register - register receive handler
3932  *      @dev: device to register a handler for
3933  *      @rx_handler: receive handler to register
3934  *      @rx_handler_data: data pointer that is used by rx handler
3935  *
3936  *      Register a receive handler for a device. This handler will then be
3937  *      called from __netif_receive_skb. A negative errno code is returned
3938  *      on a failure.
3939  *
3940  *      The caller must hold the rtnl_mutex.
3941  *
3942  *      For a general description of rx_handler, see enum rx_handler_result.
3943  */
3944 int netdev_rx_handler_register(struct net_device *dev,
3945                                rx_handler_func_t *rx_handler,
3946                                void *rx_handler_data)
3947 {
3948         ASSERT_RTNL();
3949 
3950         if (dev->rx_handler)
3951                 return -EBUSY;
3952 
3953         /* Note: rx_handler_data must be set before rx_handler */
3954         rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
3955         rcu_assign_pointer(dev->rx_handler, rx_handler);
3956 
3957         return 0;
3958 }
3959 EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
3960 
3961 /**
3962  *      netdev_rx_handler_unregister - unregister receive handler
3963  *      @dev: device to unregister a handler from
3964  *
3965  *      Unregister a receive handler from a device.
3966  *
3967  *      The caller must hold the rtnl_mutex.
3968  */
3969 void netdev_rx_handler_unregister(struct net_device *dev)
3970 {
3971 
3972         ASSERT_RTNL();
3973         RCU_INIT_POINTER(dev->rx_handler, NULL);
3974         /* a reader seeing a non NULL rx_handler in a rcu_read_lock()
3975          * section has a guarantee to see a non NULL rx_handler_data
3976          * as well.
3977          */
3978         synchronize_net();
3979         RCU_INIT_POINTER(dev->rx_handler_data, NULL);
3980 }
3981 EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
3982 
3983 /*
3984  * Limit the use of PFMEMALLOC reserves to those protocols that implement
3985  * the special handling of PFMEMALLOC skbs.
3986  */
3987 static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
3988 {
3989         switch (skb->protocol) {
3990         case htons(ETH_P_ARP):
3991         case htons(ETH_P_IP):
3992         case htons(ETH_P_IPV6):
3993         case htons(ETH_P_8021Q):
3994         case htons(ETH_P_8021AD):
3995                 return true;
3996         default:
3997                 return false;
3998         }
3999 }
4000 
4001 static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev,
4002                              int *ret, struct net_device *orig_dev)
4003 {
4004 #ifdef CONFIG_NETFILTER_INGRESS
4005         if (nf_hook_ingress_active(skb)) {
4006                 if (*pt_prev) {
4007                         *ret = deliver_skb(skb, *pt_prev, orig_dev);
4008                         *pt_prev = NULL;
4009                 }
4010 
4011                 return nf_hook_ingress(skb);
4012         }
4013 #endif /* CONFIG_NETFILTER_INGRESS */
4014         return 0;
4015 }
4016 
4017 static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
4018 {
4019         struct packet_type *ptype, *pt_prev;
4020         rx_handler_func_t *rx_handler;
4021         struct net_device *orig_dev;
4022         bool deliver_exact = false;
4023         int ret = NET_RX_DROP;
4024         __be16 type;
4025 
4026         net_timestamp_check(!netdev_tstamp_prequeue, skb);
4027 
4028         trace_netif_receive_skb(skb);
4029 
4030         orig_dev = skb->dev;
4031 
4032         skb_reset_network_header(skb);
4033         if (!skb_transport_header_was_set(skb))
4034                 skb_reset_transport_header(skb);
4035         skb_reset_mac_len(skb);
4036 
4037         pt_prev = NULL;
4038 
4039 another_round:
4040         skb->skb_iif = skb->dev->ifindex;
4041 
4042         __this_cpu_inc(softnet_data.processed);
4043 
4044         if (skb->protocol == cpu_to_be16(ETH_P_8021Q) ||
4045             skb->protocol == cpu_to_be16(ETH_P_8021AD)) {
4046                 skb = skb_vlan_untag(skb);
4047                 if (unlikely(!skb))
4048                         goto out;
4049         }
4050 
4051 #ifdef CONFIG_NET_CLS_ACT
4052         if (skb->tc_verd & TC_NCLS) {
4053                 skb->tc_verd = CLR_TC_NCLS(skb->tc_verd);
4054                 goto ncls;
4055         }
4056 #endif
4057 
4058         if (pfmemalloc)
4059                 goto skip_taps;
4060 
4061         list_for_each_entry_rcu(ptype, &ptype_all, list) {
4062                 if (pt_prev)
4063                         ret = deliver_skb(skb, pt_prev, orig_dev);
4064                 pt_prev = ptype;
4065         }
4066 
4067         list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
4068                 if (pt_prev)
4069                         ret = deliver_skb(skb, pt_prev, orig_dev);
4070                 pt_prev = ptype;
4071         }
4072 
4073 skip_taps:
4074 #ifdef CONFIG_NET_INGRESS
4075         if (static_key_false(&ingress_needed)) {
4076                 skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev);
4077                 if (!skb)
4078                         goto out;
4079 
4080                 if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0)
4081                         goto out;
4082         }
4083 #endif
4084 #ifdef CONFIG_NET_CLS_ACT
4085         skb->tc_verd = 0;
4086 ncls:
4087 #endif
4088         if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
4089                 goto drop;
4090 
4091         if (skb_vlan_tag_present(skb)) {
4092                 if (pt_prev) {
4093                         ret = deliver_skb(skb, pt_prev, orig_dev);
4094                         pt_prev = NULL;
4095                 }
4096                 if (vlan_do_receive(&skb))
4097                         goto another_round;
4098                 else if (unlikely(!skb))
4099                         goto out;
4100         }
4101 
4102         rx_handler = rcu_dereference(skb->dev->rx_handler);
4103         if (rx_handler) {
4104                 if (pt_prev) {
4105                         ret = deliver_skb(skb, pt_prev, orig_dev);
4106                         pt_prev = NULL;
4107                 }
4108                 switch (rx_handler(&skb)) {
4109                 case RX_HANDLER_CONSUMED:
4110                         ret = NET_RX_SUCCESS;
4111                         goto out;
4112                 case RX_HANDLER_ANOTHER:
4113                         goto another_round;
4114                 case RX_HANDLER_EXACT:
4115                         deliver_exact = true;
4116                 case RX_HANDLER_PASS:
4117                         break;
4118                 default:
4119                         BUG();
4120                 }
4121         }
4122 
4123         if (unlikely(skb_vlan_tag_present(skb))) {
4124                 if (skb_vlan_tag_get_id(skb))
4125                         skb->pkt_type = PACKET_OTHERHOST;
4126                 /* Note: we might in the future use prio bits
4127                  * and set skb->priority like in vlan_do_receive()
4128                  * For the time being, just ignore Priority Code Point
4129                  */
4130                 skb->vlan_tci = 0;
4131         }
4132 
4133         type = skb->protocol;
4134 
4135         /* deliver only exact match when indicated */
4136         if (likely(!deliver_exact)) {
4137                 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4138                                        &ptype_base[ntohs(type) &
4139                                                    PTYPE_HASH_MASK]);
4140         }
4141 
4142         deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4143                                &orig_dev->ptype_specific);
4144 
4145         if (unlikely(skb->dev != orig_dev)) {
4146                 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4147                                        &skb->dev->ptype_specific);
4148         }
4149 
4150         if (pt_prev) {
4151                 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
4152                         goto drop;
4153                 else
4154                         ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
4155         } else {
4156 drop:
4157                 atomic_long_inc(&skb->dev->rx_dropped);
4158                 kfree_skb(skb);
4159                 /* Jamal, now you will not able to escape explaining
4160                  * me how you were going to use this. :-)
4161                  */
4162                 ret = NET_RX_DROP;
4163         }
4164 
4165 out:
4166         return ret;
4167 }
4168 
4169 static int __netif_receive_skb(struct sk_buff *skb)
4170 {
4171         int ret;
4172 
4173         if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
4174                 unsigned long pflags = current->flags;
4175 
4176                 /*
4177                  * PFMEMALLOC skbs are special, they should
4178                  * - be delivered to SOCK_MEMALLOC sockets only
4179                  * - stay away from userspace
4180                  * - have bounded memory usage
4181                  *
4182                  * Use PF_MEMALLOC as this saves us from propagating the allocation
4183                  * context down to all allocation sites.
4184                  */
4185                 current->flags |= PF_MEMALLOC;
4186                 ret = __netif_receive_skb_core(skb, true);
4187                 tsk_restore_flags(current, pflags, PF_MEMALLOC);
4188         } else
4189                 ret = __netif_receive_skb_core(skb, false);
4190 
4191         return ret;
4192 }
4193 
4194 static int netif_receive_skb_internal(struct sk_buff *skb)
4195 {
4196         int ret;
4197 
4198         net_timestamp_check(netdev_tstamp_prequeue, skb);
4199 
4200         if (skb_defer_rx_timestamp(skb))
4201                 return NET_RX_SUCCESS;
4202 
4203         rcu_read_lock();
4204 
4205 #ifdef CONFIG_RPS
4206         if (static_key_false(&rps_needed)) {
4207                 struct rps_dev_flow voidflow, *rflow = &voidflow;
4208                 int cpu = get_rps_cpu(skb->dev, skb, &rflow);
4209 
4210                 if (cpu >= 0) {
4211                         ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
4212                         rcu_read_unlock();
4213                         return ret;
4214                 }
4215         }
4216 #endif
4217         ret = __netif_receive_skb(skb);
4218         rcu_read_unlock();
4219         return ret;
4220 }
4221 
4222 /**
4223  *      netif_receive_skb - process receive buffer from network
4224  *      @skb: buffer to process
4225  *
4226  *      netif_receive_skb() is the main receive data processing function.
4227  *      It always succeeds. The buffer may be dropped during processing
4228  *      for congestion control or by the protocol layers.
4229  *
4230  *      This function may only be called from softirq context and interrupts
4231  *      should be enabled.
4232  *
4233  *      Return values (usually ignored):
4234  *      NET_RX_SUCCESS: no congestion
4235  *      NET_RX_DROP: packet was dropped
4236  */
4237 int netif_receive_skb(struct sk_buff *skb)
4238 {
4239         trace_netif_receive_skb_entry(skb);
4240 
4241         return netif_receive_skb_internal(skb);
4242 }
4243 EXPORT_SYMBOL(netif_receive_skb);
4244 
4245 /* Network device is going away, flush any packets still pending
4246  * Called with irqs disabled.
4247  */
4248 static void flush_backlog(void *arg)
4249 {
4250         struct net_device *dev = arg;
4251         struct softnet_data *sd = this_cpu_ptr(&softnet_data);
4252         struct sk_buff *skb, *tmp;
4253 
4254         rps_lock(sd);
4255         skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
4256                 if (skb->dev == dev) {
4257                         __skb_unlink(skb, &sd->input_pkt_queue);
4258                         kfree_skb(skb);
4259                         input_queue_head_incr(sd);
4260                 }
4261         }
4262         rps_unlock(sd);
4263 
4264         skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
4265                 if (skb->dev == dev) {
4266                         __skb_unlink(skb, &sd->process_queue);
4267                         kfree_skb(skb);
4268                         input_queue_head_incr(sd);
4269                 }
4270         }
4271 }
4272 
4273 static int napi_gro_complete(struct sk_buff *skb)
4274 {
4275         struct packet_offload *ptype;
4276         __be16 type = skb->protocol;
4277         struct list_head *head = &offload_base;
4278         int err = -ENOENT;
4279 
4280         BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));
4281 
4282         if (NAPI_GRO_CB(skb)->count == 1) {
4283                 skb_shinfo(skb)->gso_size = 0;
4284                 goto out;
4285         }
4286 
4287         rcu_read_lock();
4288         list_for_each_entry_rcu(ptype, head, list) {
4289                 if (ptype->type != type || !ptype->callbacks.gro_complete)
4290                         continue;
4291 
4292                 err = ptype->callbacks.gro_complete(skb, 0);
4293                 break;
4294         }
4295         rcu_read_unlock();
4296 
4297         if (err) {
4298                 WARN_ON(&ptype->list == head);
4299                 kfree_skb(skb);
4300                 return NET_RX_SUCCESS;
4301         }
4302 
4303 out:
4304         return netif_receive_skb_internal(skb);
4305 }
4306 
4307 /* napi->gro_list contains packets ordered by age.
4308  * youngest packets at the head of it.
4309  * Complete skbs in reverse order to reduce latencies.
4310  */
4311 void napi_gro_flush(struct napi_struct *napi, bool flush_old)
4312 {
4313         struct sk_buff *skb, *prev = NULL;
4314 
4315         /* scan list and build reverse chain */
4316         for (skb = napi->gro_list; skb != NULL; skb = skb->next) {
4317                 skb->prev = prev;
4318                 prev = skb;
4319         }
4320 
4321         for (skb = prev; skb; skb = prev) {
4322                 skb->next = NULL;
4323 
4324                 if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
4325                         return;
4326 
4327                 prev = skb->prev;
4328                 napi_gro_complete(skb);
4329                 napi->gro_count--;
4330         }
4331 
4332         napi->gro_list = NULL;
4333 }
4334 EXPORT_SYMBOL(napi_gro_flush);
4335 
4336 static void gro_list_prepare(struct napi_struct *napi, struct sk_buff *skb)
4337 {
4338         struct sk_buff *p;
4339         unsigned int maclen = skb->dev->hard_header_len;
4340         u32 hash = skb_get_hash_raw(skb);
4341 
4342         for (p = napi->gro_list; p; p = p->next) {
4343                 unsigned long diffs;
4344 
4345                 NAPI_GRO_CB(p)->flush = 0;
4346 
4347                 if (hash != skb_get_hash_raw(p)) {
4348                         NAPI_GRO_CB(p)->same_flow = 0;
4349                         continue;
4350                 }
4351 
4352                 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
4353                 diffs |= p->vlan_tci ^ skb->vlan_tci;
4354                 diffs |= skb_metadata_dst_cmp(p, skb);
4355                 if (maclen == ETH_HLEN)
4356                         diffs |= compare_ether_header(skb_mac_header(p),
4357                                                       skb_mac_header(skb));
4358                 else if (!diffs)
4359                         diffs = memcmp(skb_mac_header(p),
4360                                        skb_mac_header(skb),
4361                                        maclen);
4362                 NAPI_GRO_CB(p)->same_flow = !diffs;
4363         }
4364 }
4365 
4366 static void skb_gro_reset_offset(struct sk_buff *skb)
4367 {
4368         const struct skb_shared_info *pinfo = skb_shinfo(skb);
4369         const skb_frag_t *frag0 = &pinfo->frags[0];
4370 
4371         NAPI_GRO_CB(skb)->data_offset = 0;
4372         NAPI_GRO_CB(skb)->frag0 = NULL;
4373         NAPI_GRO_CB(skb)->frag0_len = 0;
4374 
4375         if (skb_mac_header(skb) == skb_tail_pointer(skb) &&
4376             pinfo->nr_frags &&
4377             !PageHighMem(skb_frag_page(frag0))) {
4378                 NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
4379                 NAPI_GRO_CB(skb)->frag0_len = skb_frag_size(frag0);
4380         }
4381 }
4382 
4383 static void gro_pull_from_frag0(struct sk_buff *skb, int grow)
4384 {
4385         struct skb_shared_info *pinfo = skb_shinfo(skb);
4386 
4387         BUG_ON(skb->end - skb->tail < grow);
4388 
4389         memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
4390 
4391         skb->data_len -= grow;
4392         skb->tail += grow;
4393 
4394         pinfo->frags[0].page_offset += grow;
4395         skb_frag_size_sub(&pinfo->frags[0], grow);
4396 
4397         if (unlikely(!skb_frag_size(&pinfo->frags[0]))) {
4398                 skb_frag_unref(skb, 0);
4399                 memmove(pinfo->frags, pinfo->frags + 1,
4400                         --pinfo->nr_frags * sizeof(pinfo->frags[0]));
4401         }
4402 }
4403 
4404 static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
4405 {
4406         struct sk_buff **pp = NULL;
4407         struct packet_offload *ptype;
4408         __be16 type = skb->protocol;
4409         struct list_head *head = &offload_base;
4410         int same_flow;
4411         enum gro_result ret;
4412         int grow;
4413 
4414         if (!(skb->dev->features & NETIF_F_GRO))
4415                 goto normal;
4416 
4417         if (skb_is_gso(skb) || skb_has_frag_list(skb) || skb->csum_bad)
4418                 goto normal;
4419 
4420         gro_list_prepare(napi, skb);
4421 
4422         rcu_read_lock();
4423         list_for_each_entry_rcu(ptype, head, list) {
4424                 if (ptype->type != type || !ptype->callbacks.gro_receive)
4425                         continue;
4426 
4427                 skb_set_network_header(skb, skb_gro_offset(skb));
4428                 skb_reset_mac_len(skb);
4429                 NAPI_GRO_CB(skb)->same_flow = 0;
4430                 NAPI_GRO_CB(skb)->flush = 0;
4431                 NAPI_GRO_CB(skb)->free = 0;
4432                 NAPI_GRO_CB(skb)->udp_mark = 0;
4433                 NAPI_GRO_CB(skb)->gro_remcsum_start = 0;
4434 
4435                 /* Setup for GRO checksum validation */
4436                 switch (skb->ip_summed) {
4437                 case CHECKSUM_COMPLETE:
4438                         NAPI_GRO_CB(skb)->csum = skb->csum;
4439                         NAPI_GRO_CB(skb)->csum_valid = 1;
4440                         NAPI_GRO_CB(skb)->csum_cnt = 0;
4441                         break;
4442                 case CHECKSUM_UNNECESSARY:
4443                         NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1;
4444                         NAPI_GRO_CB(skb)->csum_valid = 0;
4445                         break;
4446                 default:
4447                         NAPI_GRO_CB(skb)->csum_cnt = 0;
4448                         NAPI_GRO_CB(skb)->csum_valid = 0;
4449                 }
4450 
4451                 pp = ptype->callbacks.gro_receive(&napi->gro_list, skb);
4452                 break;
4453         }
4454         rcu_read_unlock();
4455 
4456         if (&ptype->list == head)
4457                 goto normal;
4458 
4459         same_flow = NAPI_GRO_CB(skb)->same_flow;
4460         ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
4461 
4462         if (pp) {
4463                 struct sk_buff *nskb = *pp;
4464 
4465                 *pp = nskb->next;
4466                 nskb->next = NULL;
4467                 napi_gro_complete(nskb);
4468                 napi->gro_count--;
4469         }
4470 
4471         if (same_flow)
4472                 goto ok;
4473 
4474         if (NAPI_GRO_CB(skb)->flush)
4475                 goto normal;
4476 
4477         if (unlikely(napi->gro_count >= MAX_GRO_SKBS)) {
4478                 struct sk_buff *nskb = napi->gro_list;
4479 
4480                 /* locate the end of the list to select the 'oldest' flow */
4481                 while (nskb->next) {
4482                         pp = &nskb->next;
4483                         nskb = *pp;
4484                 }
4485                 *pp = NULL;
4486                 nskb->next = NULL;
4487                 napi_gro_complete(nskb);
4488         } else {
4489                 napi->gro_count++;
4490         }
4491         NAPI_GRO_CB(skb)->count = 1;
4492         NAPI_GRO_CB(skb)->age = jiffies;
4493         NAPI_GRO_CB(skb)->last = skb;
4494         skb_shinfo(skb)->gso_size = skb_gro_len(skb);
4495         skb->next = napi->gro_list;
4496         napi->gro_list = skb;
4497         ret = GRO_HELD;
4498 
4499 pull:
4500         grow = skb_gro_offset(skb) - skb_headlen(skb);
4501         if (grow > 0)
4502                 gro_pull_from_frag0(skb, grow);
4503 ok:
4504         return ret;
4505 
4506 normal:
4507         ret = GRO_NORMAL;
4508         goto pull;
4509 }
4510 
4511 struct packet_offload *gro_find_receive_by_type(__be16 type)
4512 {
4513         struct list_head *offload_head = &offload_base;
4514         struct packet_offload *ptype;
4515 
4516         list_for_each_entry_rcu(ptype, offload_head, list) {
4517                 if (ptype->type != type || !ptype->callbacks.gro_receive)
4518                         continue;
4519                 return ptype;
4520         }
4521         return NULL;
4522 }
4523 EXPORT_SYMBOL(gro_find_receive_by_type);
4524 
4525 struct packet_offload *gro_find_complete_by_type(__be16 type)
4526 {
4527         struct list_head *offload_head = &offload_base;
4528         struct packet_offload *ptype;
4529 
4530         list_for_each_entry_rcu(ptype, offload_head, list) {
4531                 if (ptype->type != type || !ptype->callbacks.gro_complete)
4532                         continue;
4533                 return ptype;
4534         }
4535         return NULL;
4536 }
4537 EXPORT_SYMBOL(gro_find_complete_by_type);
4538 
4539 static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
4540 {
4541         switch (ret) {
4542         case GRO_NORMAL:
4543                 if (netif_receive_skb_internal(skb))
4544                         ret = GRO_DROP;
4545                 break;
4546 
4547         case GRO_DROP:
4548                 kfree_skb(skb);
4549                 break;
4550 
4551         case GRO_MERGED_FREE:
4552                 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) {
4553                         skb_dst_drop(skb);
4554                         kmem_cache_free(skbuff_head_cache, skb);
4555                 } else {
4556                         __kfree_skb(skb);
4557                 }
4558                 break;
4559 
4560         case GRO_HELD:
4561         case GRO_MERGED:
4562                 break;
4563         }
4564 
4565         return ret;
4566 }
4567 
4568 gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
4569 {
4570         skb_mark_napi_id(skb, napi);
4571         trace_napi_gro_receive_entry(skb);
4572 
4573         skb_gro_reset_offset(skb);
4574 
4575         return napi_skb_finish(dev_gro_receive(napi, skb), skb);
4576 }
4577 EXPORT_SYMBOL(napi_gro_receive);
4578 
4579 static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
4580 {
4581         if (unlikely(skb->pfmemalloc)) {
4582                 consume_skb(skb);
4583                 return;
4584         }
4585         __skb_pull(skb, skb_headlen(skb));
4586         /* restore the reserve we had after netdev_alloc_skb_ip_align() */
4587         skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
4588         skb->vlan_tci = 0;
4589         skb->dev = napi->dev;
4590         skb->skb_iif = 0;
4591         skb->encapsulation = 0;
4592         skb_shinfo(skb)->gso_type = 0;
4593         skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
4594 
4595         napi->skb = skb;
4596 }
4597 
4598 struct sk_buff *napi_get_frags(struct napi_struct *napi)
4599 {
4600         struct sk_buff *skb = napi->skb;
4601 
4602         if (!skb) {
4603                 skb = napi_alloc_skb(napi, GRO_MAX_HEAD);
4604                 if (skb) {
4605                         napi->skb = skb;
4606                         skb_mark_napi_id(skb, napi);
4607                 }
4608         }
4609         return skb;
4610 }
4611 EXPORT_SYMBOL(napi_get_frags);
4612 
4613 static gro_result_t napi_frags_finish(struct napi_struct *napi,
4614                                       struct sk_buff *skb,
4615                                       gro_result_t ret)
4616 {
4617         switch (ret) {
4618         case GRO_NORMAL:
4619         case GRO_HELD:
4620                 __skb_push(skb, ETH_HLEN);
4621                 skb->protocol = eth_type_trans(skb, skb->dev);
4622                 if (ret == GRO_NORMAL && netif_receive_skb_internal(skb))
4623                         ret = GRO_DROP;
4624                 break;
4625 
4626         case GRO_DROP:
4627         case GRO_MERGED_FREE:
4628                 napi_reuse_skb(napi, skb);
4629                 break;
4630 
4631         case GRO_MERGED:
4632                 break;
4633         }
4634 
4635         return ret;
4636 }
4637 
4638 /* Upper GRO stack assumes network header starts at gro_offset=0
4639  * Drivers could call both napi_gro_frags() and napi_gro_receive()
4640  * We copy ethernet header into skb->data to have a common layout.
4641  */
4642 static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
4643 {
4644         struct sk_buff *skb = napi->skb;
4645         const struct ethhdr *eth;
4646         unsigned int hlen = sizeof(*eth);
4647 
4648         napi->skb = NULL;
4649 
4650         skb_reset_mac_header(skb);
4651         skb_gro_reset_offset(skb);
4652 
4653         eth = skb_gro_header_fast(skb, 0);
4654         if (unlikely(skb_gro_header_hard(skb, hlen))) {
4655                 eth = skb_gro_header_slow(skb, hlen, 0);
4656                 if (unlikely(!eth)) {
4657                         napi_reuse_skb(napi, skb);
4658                         return NULL;
4659                 }
4660         } else {
4661                 gro_pull_from_frag0(skb, hlen);
4662                 NAPI_GRO_CB(skb)->frag0 += hlen;
4663                 NAPI_GRO_CB(skb)->frag0_len -= hlen;
4664         }
4665         __skb_pull(skb, hlen);
4666 
4667         /*
4668          * This works because the only protocols we care about don't require
4669          * special handling.
4670          * We'll fix it up properly in napi_frags_finish()
4671          */
4672         skb->protocol = eth->h_proto;
4673 
4674         return skb;
4675 }
4676 
4677 gro_result_t napi_gro_frags(struct napi_struct *napi)
4678 {
4679         struct sk_buff *skb = napi_frags_skb(napi);
4680 
4681         if (!skb)
4682                 return GRO_DROP;
4683 
4684         trace_napi_gro_frags_entry(skb);
4685 
4686         return napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));
4687 }
4688 EXPORT_SYMBOL(napi_gro_frags);
4689 
4690 /* Compute the checksum from gro_offset and return the folded value
4691  * after adding in any pseudo checksum.
4692  */
4693 __sum16 __skb_gro_checksum_complete(struct sk_buff *skb)
4694 {
4695         __wsum wsum;
4696         __sum16 sum;
4697 
4698         wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0);
4699 
4700         /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */
4701         sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum));
4702         if (likely(!sum)) {
4703                 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
4704                     !skb->csum_complete_sw)
4705                         netdev_rx_csum_fault(skb->dev);
4706         }
4707 
4708         NAPI_GRO_CB(skb)->csum = wsum;
4709         NAPI_GRO_CB(skb)->csum_valid = 1;
4710 
4711         return sum;
4712 }
4713 EXPORT_SYMBOL(__skb_gro_checksum_complete);
4714 
4715 /*
4716  * net_rps_action_and_irq_enable sends any pending IPI's for rps.
4717  * Note: called with local irq disabled, but exits with local irq enabled.
4718  */
4719 static void net_rps_action_and_irq_enable(struct softnet_data *sd)
4720 {
4721 #ifdef CONFIG_RPS
4722         struct softnet_data *remsd = sd->rps_ipi_list;
4723 
4724         if (remsd) {
4725                 sd->rps_ipi_list = NULL;
4726 
4727                 local_irq_enable();
4728 
4729                 /* Send pending IPI's to kick RPS processing on remote cpus. */
4730                 while (remsd) {
4731                         struct softnet_data *next = remsd->rps_ipi_next;
4732 
4733                         if (cpu_online(remsd->cpu))
4734                                 smp_call_function_single_async(remsd->cpu,
4735                                                            &remsd->csd);
4736                         remsd = next;
4737                 }
4738         } else
4739 #endif
4740                 local_irq_enable();
4741 }
4742 
4743 static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
4744 {
4745 #ifdef CONFIG_RPS
4746         return sd->rps_ipi_list != NULL;
4747 #else
4748         return false;
4749 #endif
4750 }
4751 
4752 static int process_backlog(struct napi_struct *napi, int quota)
4753 {
4754         int work = 0;
4755         struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
4756 
4757         /* Check if we have pending ipi, its better to send them now,
4758          * not waiting net_rx_action() end.
4759          */
4760         if (sd_has_rps_ipi_waiting(sd)) {
4761                 local_irq_disable();
4762                 net_rps_action_and_irq_enable(sd);
4763         }
4764 
4765         napi->weight = weight_p;
4766         local_irq_disable();
4767         while (1) {
4768                 struct sk_buff *skb;
4769 
4770                 while ((skb = __skb_dequeue(&sd->process_queue))) {
4771                         rcu_read_lock();
4772                         local_irq_enable();
4773                         __netif_receive_skb(skb);
4774                         rcu_read_unlock();
4775                         local_irq_disable();
4776                         input_queue_head_incr(sd);
4777                         if (++work >= quota) {
4778                                 local_irq_enable();
4779                                 return work;
4780                         }
4781                 }
4782 
4783                 rps_lock(sd);
4784                 if (skb_queue_empty(&sd->input_pkt_queue)) {
4785                         /*
4786                          * Inline a custom version of __napi_complete().
4787                          * only current cpu owns and manipulates this napi,
4788                          * and NAPI_STATE_SCHED is the only possible flag set
4789                          * on backlog.
4790                          * We can use a plain write instead of clear_bit(),
4791                          * and we dont need an smp_mb() memory barrier.
4792                          */
4793                         napi->state = 0;
4794                         rps_unlock(sd);
4795 
4796                         break;
4797                 }
4798 
4799                 skb_queue_splice_tail_init(&sd->input_pkt_queue,
4800                                            &sd->process_queue);
4801                 rps_unlock(sd);
4802         }
4803         local_irq_enable();
4804 
4805         return work;
4806 }
4807 
4808 /**
4809  * __napi_schedule - schedule for receive
4810  * @n: entry to schedule
4811  *
4812  * The entry's receive function will be scheduled to run.
4813  * Consider using __napi_schedule_irqoff() if hard irqs are masked.
4814  */
4815 void __napi_schedule(struct napi_struct *n)
4816 {
4817         unsigned long flags;
4818 
4819         local_irq_save(flags);
4820         ____napi_schedule(this_cpu_ptr(&softnet_data), n);
4821         local_irq_restore(flags);
4822 }
4823 EXPORT_SYMBOL(__napi_schedule);
4824 
4825 /**
4826  * __napi_schedule_irqoff - schedule for receive
4827  * @n: entry to schedule
4828  *
4829  * Variant of __napi_schedule() assuming hard irqs are masked
4830  */
4831 void __napi_schedule_irqoff(struct napi_struct *n)
4832 {
4833         ____napi_schedule(this_cpu_ptr(&softnet_data), n);
4834 }
4835 EXPORT_SYMBOL(__napi_schedule_irqoff);
4836 
4837 void __napi_complete(struct napi_struct *n)
4838 {
4839         BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
4840 
4841         list_del_init(&n->poll_list);
4842         smp_mb__before_atomic();
4843         clear_bit(NAPI_STATE_SCHED, &n->state);
4844 }
4845 EXPORT_SYMBOL(__napi_complete);
4846 
4847 void napi_complete_done(struct napi_struct *n, int work_done)
4848 {
4849         unsigned long flags;
4850 
4851         /*
4852          * don't let napi dequeue from the cpu poll list
4853          * just in case its running on a different cpu
4854          */
4855         if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state)))
4856                 return;
4857 
4858         if (n->gro_list) {
4859                 unsigned long timeout = 0;
4860 
4861                 if (work_done)
4862                         timeout = n->dev->gro_flush_timeout;
4863 
4864                 if (timeout)
4865                         hrtimer_start(&n->timer, ns_to_ktime(timeout),
4866                                       HRTIMER_MODE_REL_PINNED);
4867                 else
4868                         napi_gro_flush(n, false);
4869         }
4870         if (likely(list_empty(&n->poll_list))) {
4871                 WARN_ON_ONCE(!test_and_clear_bit(NAPI_STATE_SCHED, &n->state));
4872         } else {
4873                 /* If n->poll_list is not empty, we need to mask irqs */
4874                 local_irq_save(flags);
4875                 __napi_complete(n);
4876                 local_irq_restore(flags);
4877         }
4878 }
4879 EXPORT_SYMBOL(napi_complete_done);
4880 
4881 /* must be called under rcu_read_lock(), as we dont take a reference */
4882 static struct napi_struct *napi_by_id(unsigned int napi_id)
4883 {
4884         unsigned int hash = napi_id % HASH_SIZE(napi_hash);
4885         struct napi_struct *napi;
4886 
4887         hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
4888                 if (napi->napi_id == napi_id)
4889                         return napi;
4890 
4891         return NULL;
4892 }
4893 
4894 #if defined(CONFIG_NET_RX_BUSY_POLL)
4895 #define BUSY_POLL_BUDGET 8
4896 bool sk_busy_loop(struct sock *sk, int nonblock)
4897 {
4898         unsigned long end_time = !nonblock ? sk_busy_loop_end_time(sk) : 0;
4899         int (*busy_poll)(struct napi_struct *dev);
4900         struct napi_struct *napi;
4901         int rc = false;
4902 
4903         rcu_read_lock();
4904 
4905         napi = napi_by_id(sk->sk_napi_id);
4906         if (!napi)
4907                 goto out;
4908 
4909         /* Note: ndo_busy_poll method is optional in linux-4.5 */
4910         busy_poll = napi->dev->netdev_ops->ndo_busy_poll;
4911 
4912         do {
4913                 rc = 0;
4914                 local_bh_disable();
4915                 if (busy_poll) {
4916                         rc = busy_poll(napi);
4917                 } else if (napi_schedule_prep(napi)) {
4918                         void *have = netpoll_poll_lock(napi);
4919 
4920                         if (test_bit(NAPI_STATE_SCHED, &napi->state)) {
4921                                 rc = napi->poll(napi, BUSY_POLL_BUDGET);
4922                                 trace_napi_poll(napi);
4923                                 if (rc == BUSY_POLL_BUDGET) {
4924                                         napi_complete_done(napi, rc);
4925                                         napi_schedule(napi);
4926                                 }
4927                         }
4928                         netpoll_poll_unlock(have);
4929                 }
4930                 if (rc > 0)
4931                         NET_ADD_STATS_BH(sock_net(sk),
4932                                          LINUX_MIB_BUSYPOLLRXPACKETS, rc);
4933                 local_bh_enable();
4934 
4935                 if (rc == LL_FLUSH_FAILED)
4936                         break; /* permanent failure */
4937 
4938                 cpu_relax();
4939         } while (!nonblock && skb_queue_empty(&sk->sk_receive_queue) &&
4940                  !need_resched() && !busy_loop_timeout(end_time));
4941 
4942         rc = !skb_queue_empty(&sk->sk_receive_queue);
4943 out:
4944         rcu_read_unlock();
4945         return rc;
4946 }
4947 EXPORT_SYMBOL(sk_busy_loop);
4948 
4949 #endif /* CONFIG_NET_RX_BUSY_POLL */
4950 
4951 void napi_hash_add(struct napi_struct *napi)
4952 {
4953         if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state) ||
4954             test_and_set_bit(NAPI_STATE_HASHED, &napi->state))
4955                 return;
4956 
4957         spin_lock(&napi_hash_lock);
4958 
4959         /* 0..NR_CPUS+1 range is reserved for sender_cpu use */
4960         do {
4961                 if (unlikely(++napi_gen_id < NR_CPUS + 1))
4962                         napi_gen_id = NR_CPUS + 1;
4963         } while (napi_by_id(napi_gen_id));
4964         napi->napi_id = napi_gen_id;
4965 
4966         hlist_add_head_rcu(&napi->napi_hash_node,
4967                            &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
4968 
4969         spin_unlock(&napi_hash_lock);
4970 }
4971 EXPORT_SYMBOL_GPL(napi_hash_add);
4972 
4973 /* Warning : caller is responsible to make sure rcu grace period
4974  * is respected before freeing memory containing @napi
4975  */
4976 bool napi_hash_del(struct napi_struct *napi)
4977 {
4978         bool rcu_sync_needed = false;
4979 
4980         spin_lock(&napi_hash_lock);
4981 
4982         if (test_and_clear_bit(NAPI_STATE_HASHED, &napi->state)) {
4983                 rcu_sync_needed = true;
4984                 hlist_del_rcu(&napi->napi_hash_node);
4985         }
4986         spin_unlock(&napi_hash_lock);
4987         return rcu_sync_needed;
4988 }
4989 EXPORT_SYMBOL_GPL(napi_hash_del);
4990 
4991 static enum hrtimer_restart napi_watchdog(struct hrtimer *timer)
4992 {
4993         struct napi_struct *napi;
4994 
4995         napi = container_of(timer, struct napi_struct, timer);
4996         if (napi->gro_list)
4997                 napi_schedule(napi);
4998 
4999         return HRTIMER_NORESTART;
5000 }
5001 
5002 void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
5003                     int (*poll)(struct napi_struct *, int), int weight)
5004 {
5005         INIT_LIST_HEAD(&napi->poll_list);
5006         hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
5007         napi->timer.function = napi_watchdog;
5008         napi->gro_count = 0;
5009         napi->gro_list = NULL;
5010         napi->skb = NULL;
5011         napi->poll = poll;
5012         if (weight > NAPI_POLL_WEIGHT)
5013                 pr_err_once("netif_napi_add() called with weight %d on device %s\n",
5014                             weight, dev->name);
5015         napi->weight = weight;
5016         list_add(&napi->dev_list, &dev->napi_list);
5017         napi->dev = dev;
5018 #ifdef CONFIG_NETPOLL
5019         spin_lock_init(&napi->poll_lock);
5020         napi->poll_owner = -1;
5021 #endif
5022         set_bit(NAPI_STATE_SCHED, &napi->state);
5023         napi_hash_add(napi);
5024 }
5025 EXPORT_SYMBOL(netif_napi_add);
5026 
5027 void napi_disable(struct napi_struct *n)
5028 {
5029         might_sleep();
5030         set_bit(NAPI_STATE_DISABLE, &n->state);
5031 
5032         while (test_and_set_bit(NAPI_STATE_SCHED, &n->state))
5033                 msleep(1);
5034         while (test_and_set_bit(NAPI_STATE_NPSVC, &n->state))
5035                 msleep(1);
5036 
5037         hrtimer_cancel(&n->timer);
5038 
5039         clear_bit(NAPI_STATE_DISABLE, &n->state);
5040 }
5041 EXPORT_SYMBOL(napi_disable);
5042 
5043 /* Must be called in process context */
5044 void netif_napi_del(struct napi_struct *napi)
5045 {
5046         might_sleep();
5047         if (napi_hash_del(napi))
5048                 synchronize_net();
5049         list_del_init(&napi->dev_list);
5050         napi_free_frags(napi);
5051 
5052         kfree_skb_list(napi->gro_list);
5053         napi->gro_list = NULL;
5054         napi->gro_count = 0;
5055 }
5056 EXPORT_SYMBOL(netif_napi_del);
5057 
5058 static int napi_poll(struct napi_struct *n, struct list_head *repoll)
5059 {
5060         void *have;
5061         int work, weight;
5062 
5063         list_del_init(&n->poll_list);
5064 
5065         have = netpoll_poll_lock(n);
5066 
5067         weight = n->weight;
5068 
5069         /* This NAPI_STATE_SCHED test is for avoiding a race
5070          * with netpoll's poll_napi().  Only the entity which
5071          * obtains the lock and sees NAPI_STATE_SCHED set will
5072          * actually make the ->poll() call.  Therefore we avoid
5073          * accidentally calling ->poll() when NAPI is not scheduled.
5074          */
5075         work = 0;
5076         if (test_bit(NAPI_STATE_SCHED, &n->state)) {
5077                 work = n->poll(n, weight);
5078                 trace_napi_poll(n);
5079         }
5080 
5081         WARN_ON_ONCE(work > weight);
5082 
5083         if (likely(work < weight))
5084                 goto out_unlock;
5085 
5086         /* Drivers must not modify the NAPI state if they
5087          * consume the entire weight.  In such cases this code
5088          * still "owns" the NAPI instance and therefore can
5089          * move the instance around on the list at-will.
5090          */
5091         if (unlikely(napi_disable_pending(n))) {
5092                 napi_complete(n);
5093                 goto out_unlock;
5094         }
5095 
5096         if (n->gro_list) {
5097                 /* flush too old packets
5098                  * If HZ < 1000, flush all packets.
5099                  */
5100                 napi_gro_flush(n, HZ >= 1000);
5101         }
5102 
5103         /* Some drivers may have called napi_schedule
5104          * prior to exhausting their budget.
5105          */
5106         if (unlikely(!list_empty(&n->poll_list))) {
5107                 pr_warn_once("%s: Budget exhausted after napi rescheduled\n",
5108                              n->dev ? n->dev->name : "backlog");
5109                 goto out_unlock;
5110         }
5111 
5112         list_add_tail(&n->poll_list, repoll);
5113 
5114 out_unlock:
5115         netpoll_poll_unlock(have);
5116 
5117         return work;
5118 }
5119 
5120 static void net_rx_action(struct softirq_action *h)
5121 {
5122         struct softnet_data *sd = this_cpu_ptr(&softnet_data);
5123         unsigned long time_limit = jiffies + 2;
5124         int budget = netdev_budget;
5125         LIST_HEAD(list);
5126         LIST_HEAD(repoll);
5127 
5128         local_irq_disable();
5129         list_splice_init(&sd->poll_list, &list);
5130         local_irq_enable();
5131 
5132         for (;;) {
5133                 struct napi_struct *n;
5134 
5135                 if (list_empty(&list)) {
5136                         if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll))
5137                                 return;
5138                         break;
5139                 }
5140 
5141                 n = list_first_entry(&list, struct napi_struct, poll_list);
5142                 budget -= napi_poll(n, &repoll);
5143 
5144                 /* If softirq window is exhausted then punt.
5145                  * Allow this to run for 2 jiffies since which will allow
5146                  * an average latency of 1.5/HZ.
5147                  */
5148                 if (unlikely(budget <= 0 ||
5149                              time_after_eq(jiffies, time_limit))) {
5150                         sd->time_squeeze++;
5151                         break;
5152                 }
5153         }
5154 
5155         local_irq_disable();
5156 
5157         list_splice_tail_init(&sd->poll_list, &list);
5158         list_splice_tail(&repoll, &list);
5159         list_splice(&list, &sd->poll_list);
5160         if (!list_empty(&sd->poll_list))
5161                 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
5162 
5163         net_rps_action_and_irq_enable(sd);
5164 }
5165 
5166 struct netdev_adjacent {
5167         struct net_device *dev;
5168 
5169         /* upper master flag, there can only be one master device per list */
5170         bool master;
5171 
5172         /* counter for the number of times this device was added to us */
5173         u16 ref_nr;
5174 
5175         /* private field for the users */
5176         void *private;
5177 
5178         struct list_head list;
5179         struct rcu_head rcu;
5180 };
5181 
5182 static struct netdev_adjacent *__netdev_find_adj(struct net_device *adj_dev,
5183                                                  struct list_head *adj_list)
5184 {
5185         struct netdev_adjacent *adj;
5186 
5187         list_for_each_entry(adj, adj_list, list) {
5188                 if (adj->dev == adj_dev)
5189                         return adj;
5190         }
5191         return NULL;
5192 }
5193 
5194 /**
5195  * netdev_has_upper_dev - Check if device is linked to an upper device
5196  * @dev: device
5197  * @upper_dev: upper device to check
5198  *
5199  * Find out if a device is linked to specified upper device and return true
5200  * in case it is. Note that this checks only immediate upper device,
5201  * not through a complete stack of devices. The caller must hold the RTNL lock.
5202  */
5203 bool netdev_has_upper_dev(struct net_device *dev,
5204                           struct net_device *upper_dev)
5205 {
5206         ASSERT_RTNL();
5207 
5208         return __netdev_find_adj(upper_dev, &dev->all_adj_list.upper);
5209 }
5210 EXPORT_SYMBOL(netdev_has_upper_dev);
5211 
5212 /**
5213  * netdev_has_any_upper_dev - Check if device is linked to some device
5214  * @dev: device
5215  *
5216  * Find out if a device is linked to an upper device and return true in case
5217  * it is. The caller must hold the RTNL lock.
5218  */
5219 static bool netdev_has_any_upper_dev(struct net_device *dev)
5220 {
5221         ASSERT_RTNL();
5222 
5223         return !list_empty(&dev->all_adj_list.upper);
5224 }
5225 
5226 /**
5227  * netdev_master_upper_dev_get - Get master upper device
5228  * @dev: device
5229  *
5230  * Find a master upper device and return pointer to it or NULL in case
5231  * it's not there. The caller must hold the RTNL lock.
5232  */
5233 struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
5234 {
5235         struct netdev_adjacent *upper;
5236 
5237         ASSERT_RTNL();
5238 
5239         if (list_empty(&dev->adj_list.upper))
5240                 return NULL;
5241 
5242         upper = list_first_entry(&dev->adj_list.upper,
5243                                  struct netdev_adjacent, list);
5244         if (likely(upper->master))
5245                 return upper->dev;
5246         return NULL;
5247 }
5248 EXPORT_SYMBOL(netdev_master_upper_dev_get);
5249 
5250 void *netdev_adjacent_get_private(struct list_head *adj_list)
5251 {
5252         struct netdev_adjacent *adj;
5253 
5254         adj = list_entry(adj_list, struct netdev_adjacent, list);
5255 
5256         return adj->private;
5257 }
5258 EXPORT_SYMBOL(netdev_adjacent_get_private);
5259 
5260 /**
5261  * netdev_upper_get_next_dev_rcu - Get the next dev from upper list
5262  * @dev: device
5263  * @iter: list_head ** of the current position
5264  *
5265  * Gets the next device from the dev's upper list, starting from iter
5266  * position. The caller must hold RCU read lock.
5267  */
5268 struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
5269                                                  struct list_head **iter)
5270 {
5271         struct netdev_adjacent *upper;
5272 
5273         WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
5274 
5275         upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5276 
5277         if (&upper->list == &dev->adj_list.upper)
5278                 return NULL;
5279 
5280         *iter = &upper->list;
5281 
5282         return upper->dev;
5283 }
5284 EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu);
5285 
5286 /**
5287  * netdev_all_upper_get_next_dev_rcu - Get the next dev from upper list
5288  * @dev: device
5289  * @iter: list_head ** of the current position
5290  *
5291  * Gets the next device from the dev's upper list, starting from iter
5292  * position. The caller must hold RCU read lock.
5293  */
5294 struct net_device *netdev_all_upper_get_next_dev_rcu(struct net_device *dev,
5295                                                      struct list_head **iter)
5296 {
5297         struct netdev_adjacent *upper;
5298 
5299         WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
5300 
5301         upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5302 
5303         if (&upper->list == &dev->all_adj_list.upper)
5304                 return NULL;
5305 
5306         *iter = &upper->list;
5307 
5308         return upper->dev;
5309 }
5310 EXPORT_SYMBOL(netdev_all_upper_get_next_dev_rcu);
5311 
5312 /**
5313  * netdev_lower_get_next_private - Get the next ->private from the
5314  *                                 lower neighbour list
5315  * @dev: device
5316  * @iter: list_head ** of the current position
5317  *
5318  * Gets the next netdev_adjacent->private from the dev's lower neighbour
5319  * list, starting from iter position. The caller must hold either hold the
5320  * RTNL lock or its own locking that guarantees that the neighbour lower
5321  * list will remain unchanged.
5322  */
5323 void *netdev_lower_get_next_private(struct net_device *dev,
5324                                     struct list_head **iter)
5325 {
5326         struct netdev_adjacent *lower;
5327 
5328         lower = list_entry(*iter, struct netdev_adjacent, list);
5329 
5330         if (&lower->list == &dev->adj_list.lower)
5331                 return NULL;
5332 
5333         *iter = lower->list.next;
5334 
5335         return lower->private;
5336 }
5337 EXPORT_SYMBOL(netdev_lower_get_next_private);
5338 
5339 /**
5340  * netdev_lower_get_next_private_rcu - Get the next ->private from the
5341  *                                     lower neighbour list, RCU
5342  *                                     variant
5343  * @dev: device
5344  * @iter: list_head ** of the current position
5345  *
5346  * Gets the next netdev_adjacent->private from the dev's lower neighbour
5347  * list, starting from iter position. The caller must hold RCU read lock.
5348  */
5349 void *netdev_lower_get_next_private_rcu(struct net_device *dev,
5350                                         struct list_head **iter)
5351 {
5352         struct netdev_adjacent *lower;
5353 
5354         WARN_ON_ONCE(!rcu_read_lock_held());
5355 
5356         lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5357 
5358         if (&lower->list == &dev->adj_list.lower)
5359                 return NULL;
5360 
5361         *iter = &lower->list;
5362 
5363         return lower->private;
5364 }
5365 EXPORT_SYMBOL(netdev_lower_get_next_private_rcu);
5366 
5367 /**
5368  * netdev_lower_get_next - Get the next device from the lower neighbour
5369  *                         list
5370  * @dev: device
5371  * @iter: list_head ** of the current position
5372  *
5373  * Gets the next netdev_adjacent from the dev's lower neighbour
5374  * list, starting from iter position. The caller must hold RTNL lock or
5375  * its own locking that guarantees that the neighbour lower
5376  * list will remain unchanged.
5377  */
5378 void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter)
5379 {
5380         struct netdev_adjacent *lower;
5381 
5382         lower = list_entry(*iter, struct netdev_adjacent, list);
5383 
5384         if (&lower->list == &dev->adj_list.lower)
5385                 return NULL;
5386 
5387         *iter = lower->list.next;
5388 
5389         return lower->dev;
5390 }
5391 EXPORT_SYMBOL(netdev_lower_get_next);
5392 
5393 /**
5394  * netdev_lower_get_first_private_rcu - Get the first ->private from the
5395  *                                     lower neighbour list, RCU
5396  *                                     variant
5397  * @dev: device
5398  *
5399  * Gets the first netdev_adjacent->private from the dev's lower neighbour
5400  * list. The caller must hold RCU read lock.
5401  */
5402 void *netdev_lower_get_first_private_rcu(struct net_device *dev)
5403 {
5404         struct netdev_adjacent *lower;
5405 
5406         lower = list_first_or_null_rcu(&dev->adj_list.lower,
5407                         struct netdev_adjacent, list);
5408         if (lower)
5409                 return lower->private;
5410         return NULL;
5411 }
5412 EXPORT_SYMBOL(netdev_lower_get_first_private_rcu);
5413 
5414 /**
5415  * netdev_master_upper_dev_get_rcu - Get master upper device
5416  * @dev: device
5417  *
5418  * Find a master upper device and return pointer to it or NULL in case
5419  * it's not there. The caller must hold the RCU read lock.
5420  */
5421 struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
5422 {
5423         struct netdev_adjacent *upper;
5424 
5425         upper = list_first_or_null_rcu(&dev->adj_list.upper,
5426                                        struct netdev_adjacent, list);
5427         if (upper && likely(upper->master))
5428                 return upper->dev;
5429         return NULL;
5430 }
5431 EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
5432 
5433 static int netdev_adjacent_sysfs_add(struct net_device *dev,
5434                               struct net_device *adj_dev,
5435                               struct list_head *dev_list)
5436 {
5437         char linkname[IFNAMSIZ+7];
5438         sprintf(linkname, dev_list == &dev->adj_list.upper ?
5439                 "upper_%s" : "lower_%s", adj_dev->name);
5440         return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj),
5441                                  linkname);
5442 }
5443 static void netdev_adjacent_sysfs_del(struct net_device *dev,
5444                                char *name,
5445                                struct list_head *dev_list)
5446 {
5447         char linkname[IFNAMSIZ+7];
5448         sprintf(linkname, dev_list == &dev->adj_list.upper ?
5449                 "upper_%s" : "lower_%s", name);
5450         sysfs_remove_link(&(dev->dev.kobj), linkname);
5451 }
5452 
5453 static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev,
5454                                                  struct net_device *adj_dev,
5455                                                  struct list_head *dev_list)
5456 {
5457         return (dev_list == &dev->adj_list.upper ||
5458                 dev_list == &dev->adj_list.lower) &&
5459                 net_eq(dev_net(dev), dev_net(adj_dev));
5460 }
5461 
5462 static int __netdev_adjacent_dev_insert(struct net_device *dev,
5463                                         struct net_device *adj_dev,
5464                                         struct list_head *dev_list,
5465                                         void *private, bool master)
5466 {
5467         struct netdev_adjacent *adj;
5468         int ret;
5469 
5470         adj = __netdev_find_adj(adj_dev, dev_list);
5471 
5472         if (adj) {
5473                 adj->ref_nr++;
5474                 return 0;
5475         }
5476 
5477         adj = kmalloc(sizeof(*adj), GFP_KERNEL);
5478         if (!adj)
5479                 return -ENOMEM;
5480 
5481         adj->dev = adj_dev;
5482         adj->master = master;
5483         adj->ref_nr = 1;
5484         adj->private = private;
5485         dev_hold(adj_dev);
5486 
5487         pr_debug("dev_hold for %s, because of link added from %s to %s\n",
5488                  adj_dev->name, dev->name, adj_dev->name);
5489 
5490         if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) {
5491                 ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list);
5492                 if (ret)
5493                         goto free_adj;
5494         }
5495 
5496         /* Ensure that master link is always the first item in list. */
5497         if (master) {
5498                 ret = sysfs_create_link(&(dev->dev.kobj),
5499                                         &(adj_dev->dev.kobj), "master");
5500                 if (ret)
5501                         goto remove_symlinks;
5502 
5503                 list_add_rcu(&adj->list, dev_list);
5504         } else {
5505                 list_add_tail_rcu(&adj->list, dev_list);
5506         }
5507 
5508         return 0;
5509 
5510 remove_symlinks:
5511         if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
5512                 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
5513 free_adj:
5514         kfree(adj);
5515         dev_put(adj_dev);
5516 
5517         return ret;
5518 }
5519 
5520 static void __netdev_adjacent_dev_remove(struct net_device *dev,
5521                                          struct net_device *adj_dev,
5522                                          struct list_head *dev_list)
5523 {
5524         struct netdev_adjacent *adj;
5525 
5526         adj = __netdev_find_adj(adj_dev, dev_list);
5527 
5528         if (!adj) {
5529                 pr_err("tried to remove device %s from %s\n",
5530                        dev->name, adj_dev->name);
5531                 BUG();
5532         }
5533 
5534         if (adj->ref_nr > 1) {
5535                 pr_debug("%s to %s ref_nr-- = %d\n", dev->name, adj_dev->name,
5536                          adj->ref_nr-1);
5537                 adj->ref_nr--;
5538                 return;
5539         }
5540 
5541         if (adj->master)
5542                 sysfs_remove_link(&(dev->dev.kobj), "master");
5543 
5544         if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
5545                 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
5546 
5547         list_del_rcu(&adj->list);
5548         pr_debug("dev_put for %s, because link removed from %s to %s\n",
5549                  adj_dev->name, dev->name, adj_dev->name);
5550         dev_put(adj_dev);
5551         kfree_rcu(adj, rcu);
5552 }
5553 
5554 static int __netdev_adjacent_dev_link_lists(struct net_device *dev,
5555                                             struct net_device *upper_dev,
5556                                             struct list_head *up_list,
5557                                             struct list_head *down_list,
5558                                             void *private, bool master)
5559 {
5560         int ret;
5561 
5562         ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list, private,
5563                                            master);
5564         if (ret)
5565                 return ret;
5566 
5567         ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list, private,
5568                                            false);
5569         if (ret) {
5570                 __netdev_adjacent_dev_remove(dev, upper_dev, up_list);
5571                 return ret;
5572         }
5573 
5574         return 0;
5575 }
5576 
5577 static int __netdev_adjacent_dev_link(struct net_device *dev,
5578                                       struct net_device *upper_dev)
5579 {
5580         return __netdev_adjacent_dev_link_lists(dev, upper_dev,
5581                                                 &dev->all_adj_list.upper,
5582                                                 &upper_dev->all_adj_list.lower,
5583                                                 NULL, false);
5584 }
5585 
5586 static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev,
5587                                                struct net_device *upper_dev,
5588                                                struct list_head *up_list,
5589                                                struct list_head *down_list)
5590 {
5591         __netdev_adjacent_dev_remove(dev, upper_dev, up_list);
5592         __netdev_adjacent_dev_remove(upper_dev, dev, down_list);
5593 }
5594 
5595 static void __netdev_adjacent_dev_unlink(struct net_device *dev,
5596                                          struct net_device *upper_dev)
5597 {
5598         __netdev_adjacent_dev_unlink_lists(dev, upper_dev,
5599                                            &dev->all_adj_list.upper,
5600                                            &upper_dev->all_adj_list.lower);
5601 }
5602 
5603 static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev,
5604                                                 struct net_device *upper_dev,
5605                                                 void *private, bool master)
5606 {
5607         int ret = __netdev_adjacent_dev_link(dev, upper_dev);
5608 
5609         if (ret)
5610                 return ret;
5611 
5612         ret = __netdev_adjacent_dev_link_lists(dev, upper_dev,
5613                                                &dev->adj_list.upper,
5614                                                &upper_dev->adj_list.lower,
5615                                                private, master);
5616         if (ret) {
5617                 __netdev_adjacent_dev_unlink(dev, upper_dev);
5618                 return ret;
5619         }
5620 
5621         return 0;
5622 }
5623 
5624 static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev,
5625                                                    struct net_device *upper_dev)
5626 {
5627         __netdev_adjacent_dev_unlink(dev, upper_dev);
5628         __netdev_adjacent_dev_unlink_lists(dev, upper_dev,
5629                                            &dev->adj_list.upper,
5630                                            &upper_dev->adj_list.lower);
5631 }
5632 
5633 static int __netdev_upper_dev_link(struct net_device *dev,
5634                                    struct net_device *upper_dev, bool master,
5635                                    void *upper_priv, void *upper_info)
5636 {
5637         struct netdev_notifier_changeupper_info changeupper_info;
5638         struct netdev_adjacent *i, *j, *to_i, *to_j;
5639         int ret = 0;
5640 
5641         ASSERT_RTNL();
5642 
5643         if (dev == upper_dev)
5644                 return -EBUSY;
5645 
5646         /* To prevent loops, check if dev is not upper device to upper_dev. */
5647         if (__netdev_find_adj(dev, &upper_dev->all_adj_list.upper))
5648                 return -EBUSY;
5649 
5650         if (__netdev_find_adj(upper_dev, &dev->adj_list.upper))
5651                 return -EEXIST;
5652 
5653         if (master && netdev_master_upper_dev_get(dev))
5654                 return -EBUSY;
5655 
5656         changeupper_info.upper_dev = upper_dev;
5657         changeupper_info.master = master;
5658         changeupper_info.linking = true;
5659         changeupper_info.upper_info = upper_info;
5660 
5661         ret = call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, dev,
5662                                             &changeupper_info.info);
5663         ret = notifier_to_errno(ret);
5664         if (ret)
5665                 return ret;
5666 
5667         ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, upper_priv,
5668                                                    master);
5669         if (ret)
5670                 return ret;
5671 
5672         /* Now that we linked these devs, make all the upper_dev's
5673          * all_adj_list.upper visible to every dev's all_adj_list.lower an
5674          * versa, and don't forget the devices itself. All of these
5675          * links are non-neighbours.
5676          */
5677         list_for_each_entry(i, &dev->all_adj_list.lower, list) {
5678                 list_for_each_entry(j, &upper_dev->all_adj_list.upper, list) {
5679                         pr_debug("Interlinking %s with %s, non-neighbour\n",
5680                                  i->dev->name, j->dev->name);
5681                         ret = __netdev_adjacent_dev_link(i->dev, j->dev);
5682                         if (ret)
5683                                 goto rollback_mesh;
5684                 }
5685         }
5686 
5687         /* add dev to every upper_dev's upper device */
5688         list_for_each_entry(i, &upper_dev->all_adj_list.upper, list) {
5689                 pr_debug("linking %s's upper device %s with %s\n",
5690                          upper_dev->name, i->dev->name, dev->name);
5691                 ret = __netdev_adjacent_dev_link(dev, i->dev);
5692                 if (ret)
5693                         goto rollback_upper_mesh;
5694         }
5695 
5696         /* add upper_dev to every dev's lower device */
5697         list_for_each_entry(i, &dev->all_adj_list.lower, list) {
5698                 pr_debug("linking %s's lower device %s with %s\n", dev->name,
5699                          i->dev->name, upper_dev->name);
5700                 ret = __netdev_adjacent_dev_link(i->dev, upper_dev);
5701                 if (ret)
5702                         goto rollback_lower_mesh;
5703         }
5704 
5705         ret = call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, dev,
5706                                             &changeupper_info.info);
5707         ret = notifier_to_errno(ret);
5708         if (ret)
5709                 goto rollback_lower_mesh;
5710 
5711         return 0;
5712 
5713 rollback_lower_mesh:
5714         to_i = i;
5715         list_for_each_entry(i, &dev->all_adj_list.lower, list) {
5716                 if (i == to_i)
5717                         break;
5718                 __netdev_adjacent_dev_unlink(i->dev, upper_dev);
5719         }
5720 
5721         i = NULL;
5722 
5723 rollback_upper_mesh:
5724         to_i = i;
5725         list_for_each_entry(i, &upper_dev->all_adj_list.upper, list) {
5726                 if (i == to_i)
5727                         break;
5728                 __netdev_adjacent_dev_unlink(dev, i->dev);
5729         }
5730 
5731         i = j = NULL;
5732 
5733 rollback_mesh:
5734         to_i = i;
5735         to_j = j;
5736         list_for_each_entry(i, &dev->all_adj_list.lower, list) {
5737                 list_for_each_entry(j, &upper_dev->all_adj_list.upper, list) {
5738                         if (i == to_i && j == to_j)
5739                                 break;
5740                         __netdev_adjacent_dev_unlink(i->dev, j->dev);
5741                 }
5742                 if (i == to_i)
5743                         break;
5744         }
5745 
5746         __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
5747 
5748         return ret;
5749 }
5750 
5751 /**
5752  * netdev_upper_dev_link - Add a link to the upper device
5753  * @dev: device
5754  * @upper_dev: new upper device
5755  *
5756  * Adds a link to device which is upper to this one. The caller must hold
5757  * the RTNL lock. On a failure a negative errno code is returned.
5758  * On success the reference counts are adjusted and the function
5759  * returns zero.
5760  */
5761 int netdev_upper_dev_link(struct net_device *dev,
5762                           struct net_device *upper_dev)
5763 {
5764         return __netdev_upper_dev_link(dev, upper_dev, false, NULL, NULL);
5765 }
5766 EXPORT_SYMBOL(netdev_upper_dev_link);
5767 
5768 /**
5769  * netdev_master_upper_dev_link - Add a master link to the upper device
5770  * @dev: device
5771  * @upper_dev: new upper device
5772  * @upper_priv: upper device private
5773  * @upper_info: upper info to be passed down via notifier
5774  *
5775  * Adds a link to device which is upper to this one. In this case, only
5776  * one master upper device can be linked, although other non-master devices
5777  * might be linked as well. The caller must hold the RTNL lock.
5778  * On a failure a negative errno code is returned. On success the reference
5779  * counts are adjusted and the function returns zero.
5780  */
5781 int netdev_master_upper_dev_link(struct net_device *dev,
5782                                  struct net_device *upper_dev,
5783                                  void *upper_priv, void *upper_info)
5784 {
5785         return __netdev_upper_dev_link(dev, upper_dev, true,
5786                                        upper_priv, upper_info);
5787 }
5788 EXPORT_SYMBOL(netdev_master_upper_dev_link);
5789 
5790 /**
5791  * netdev_upper_dev_unlink - Removes a link to upper device
5792  * @dev: device
5793  * @upper_dev: new upper device
5794  *
5795  * Removes a link to device which is upper to this one. The caller must hold
5796  * the RTNL lock.
5797  */
5798 void netdev_upper_dev_unlink(struct net_device *dev,
5799                              struct net_device *upper_dev)
5800 {
5801         struct netdev_notifier_changeupper_info changeupper_info;
5802         struct netdev_adjacent *i, *j;
5803         ASSERT_RTNL();
5804 
5805         changeupper_info.upper_dev = upper_dev;
5806         changeupper_info.master = netdev_master_upper_dev_get(dev) == upper_dev;
5807         changeupper_info.linking = false;
5808 
5809         call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, dev,
5810                                       &changeupper_info.info);
5811 
5812         __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
5813 
5814         /* Here is the tricky part. We must remove all dev's lower
5815          * devices from all upper_dev's upper devices and vice
5816          * versa, to maintain the graph relationship.
5817          */
5818         list_for_each_entry(i, &dev->all_adj_list.lower, list)
5819                 list_for_each_entry(j, &upper_dev->all_adj_list.upper, list)
5820                         __netdev_adjacent_dev_unlink(i->dev, j->dev);
5821 
5822         /* remove also the devices itself from lower/upper device
5823          * list
5824          */
5825         list_for_each_entry(i, &dev->all_adj_list.lower, list)
5826                 __netdev_adjacent_dev_unlink(i->dev, upper_dev);
5827 
5828         list_for_each_entry(i, &upper_dev->all_adj_list.upper, list)
5829                 __netdev_adjacent_dev_unlink(dev, i->dev);
5830 
5831         call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, dev,
5832                                       &changeupper_info.info);
5833 }
5834 EXPORT_SYMBOL(netdev_upper_dev_unlink);
5835 
5836 /**
5837  * netdev_bonding_info_change - Dispatch event about slave change
5838  * @dev: device
5839  * @bonding_info: info to dispatch
5840  *
5841  * Send NETDEV_BONDING_INFO to netdev notifiers with info.
5842  * The caller must hold the RTNL lock.
5843  */
5844 void netdev_bonding_info_change(struct net_device *dev,
5845                                 struct netdev_bonding_info *bonding_info)
5846 {
5847         struct netdev_notifier_bonding_info     info;
5848 
5849         memcpy(&info.bonding_info, bonding_info,
5850                sizeof(struct netdev_bonding_info));
5851         call_netdevice_notifiers_info(NETDEV_BONDING_INFO, dev,
5852                                       &info.info);
5853 }
5854 EXPORT_SYMBOL(netdev_bonding_info_change);
5855 
5856 static void netdev_adjacent_add_links(struct net_device *dev)
5857 {
5858         struct netdev_adjacent *iter;
5859 
5860         struct net *net = dev_net(dev);
5861 
5862         list_for_each_entry(iter, &dev->adj_list.upper, list) {
5863                 if (!net_eq(net,dev_net(iter->dev)))
5864                         continue;
5865                 netdev_adjacent_sysfs_add(iter->dev, dev,
5866                                           &iter->dev->adj_list.lower);
5867                 netdev_adjacent_sysfs_add(dev, iter->dev,
5868                                           &dev->adj_list.upper);
5869         }
5870 
5871         list_for_each_entry(iter, &dev->adj_list.lower, list) {
5872                 if (!net_eq(net,dev_net(iter->dev)))
5873                         continue;
5874                 netdev_adjacent_sysfs_add(iter->dev, dev,
5875                                           &iter->dev->adj_list.upper);
5876                 netdev_adjacent_sysfs_add(dev, iter->dev,
5877                                           &dev->adj_list.lower);
5878         }
5879 }
5880 
5881 static void netdev_adjacent_del_links(struct net_device *dev)
5882 {
5883         struct netdev_adjacent *iter;
5884 
5885         struct net *net = dev_net(dev);
5886 
5887         list_for_each_entry(iter, &dev->adj_list.upper, list) {
5888                 if (!net_eq(net,dev_net(iter->dev)))
5889                         continue;
5890                 netdev_adjacent_sysfs_del(iter->dev, dev->name,
5891                                           &iter->dev->adj_list.lower);
5892                 netdev_adjacent_sysfs_del(dev, iter->dev->name,
5893                                           &dev->adj_list.upper);
5894         }
5895 
5896         list_for_each_entry(iter, &dev->adj_list.lower, list) {
5897                 if (!net_eq(net,dev_net(iter->dev)))
5898                         continue;
5899                 netdev_adjacent_sysfs_del(iter->dev, dev->name,
5900                                           &iter->dev->adj_list.upper);
5901                 netdev_adjacent_sysfs_del(dev, iter->dev->name,
5902                                           &dev->adj_list.lower);
5903         }
5904 }
5905 
5906 void netdev_adjacent_rename_links(struct net_device *dev, char *oldname)
5907 {
5908         struct netdev_adjacent *iter;
5909 
5910         struct net *net = dev_net(dev);
5911 
5912         list_for_each_entry(iter, &dev->adj_list.upper, list) {
5913                 if (!net_eq(net,dev_net(iter->dev)))
5914                         continue;
5915                 netdev_adjacent_sysfs_del(iter->dev, oldname,
5916                                           &iter->dev->adj_list.lower);
5917                 netdev_adjacent_sysfs_add(iter->dev, dev,
5918                                           &iter->dev->adj_list.lower);
5919         }
5920 
5921         list_for_each_entry(iter, &dev->adj_list.lower, list) {
5922                 if (!net_eq(net,dev_net(iter->dev)))
5923                         continue;
5924                 netdev_adjacent_sysfs_del(iter->dev, oldname,
5925                                           &iter->dev->adj_list.upper);
5926                 netdev_adjacent_sysfs_add(iter->dev, dev,
5927                                           &iter->dev->adj_list.upper);
5928         }
5929 }
5930 
5931 void *netdev_lower_dev_get_private(struct net_device *dev,
5932                                    struct net_device *lower_dev)
5933 {
5934         struct netdev_adjacent *lower;
5935 
5936         if (!lower_dev)
5937                 return NULL;
5938         lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower);
5939         if (!lower)
5940                 return NULL;
5941 
5942         return lower->private;
5943 }
5944 EXPORT_SYMBOL(netdev_lower_dev_get_private);
5945 
5946 
5947 int dev_get_nest_level(struct net_device *dev,
5948                        bool (*type_check)(const struct net_device *dev))
5949 {
5950         struct net_device *lower = NULL;
5951         struct list_head *iter;
5952         int max_nest = -1;
5953         int nest;
5954 
5955         ASSERT_RTNL();
5956 
5957         netdev_for_each_lower_dev(dev, lower, iter) {
5958                 nest = dev_get_nest_level(lower, type_check);
5959                 if (max_nest < nest)
5960                         max_nest = nest;
5961         }
5962 
5963         if (type_check(dev))
5964                 max_nest++;
5965 
5966         return max_nest;
5967 }
5968 EXPORT_SYMBOL(dev_get_nest_level);
5969 
5970 /**
5971  * netdev_lower_change - Dispatch event about lower device state change
5972  * @lower_dev: device
5973  * @lower_state_info: state to dispatch
5974  *
5975  * Send NETDEV_CHANGELOWERSTATE to netdev notifiers with info.
5976  * The caller must hold the RTNL lock.
5977  */
5978 void netdev_lower_state_changed(struct net_device *lower_dev,
5979                                 void *lower_state_info)
5980 {
5981         struct netdev_notifier_changelowerstate_info changelowerstate_info;
5982 
5983         ASSERT_RTNL();
5984         changelowerstate_info.lower_state_info = lower_state_info;
5985         call_netdevice_notifiers_info(NETDEV_CHANGELOWERSTATE, lower_dev,
5986                                       &changelowerstate_info.info);
5987 }
5988 EXPORT_SYMBOL(netdev_lower_state_changed);
5989 
5990 static void dev_change_rx_flags(struct net_device *dev, int flags)
5991 {
5992         const struct net_device_ops *ops = dev->netdev_ops;
5993 
5994         if (ops->ndo_change_rx_flags)
5995                 ops->ndo_change_rx_flags(dev, flags);
5996 }
5997 
5998 static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify)
5999 {
6000         unsigned int old_flags = dev->flags;
6001         kuid_t uid;
6002         kgid_t gid;
6003 
6004         ASSERT_RTNL();
6005 
6006         dev->flags |= IFF_PROMISC;
6007         dev->promiscuity += inc;
6008         if (dev->promiscuity == 0) {
6009                 /*
6010                  * Avoid overflow.
6011                  * If inc causes overflow, untouch promisc and return error.
6012                  */
6013                 if (inc < 0)
6014                         dev->flags &= ~IFF_PROMISC;
6015                 else {
6016                         dev->promiscuity -= inc;
6017                         pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n",
6018                                 dev->name);
6019                         return -EOVERFLOW;
6020                 }
6021         }
6022         if (dev->flags != old_flags) {
6023                 pr_info("device %s %s promiscuous mode\n",
6024                         dev->name,
6025                         dev->flags & IFF_PROMISC ? "entered" : "left");
6026                 if (audit_enabled) {
6027                         current_uid_gid(&uid, &gid);
6028                         audit_log(current->audit_context, GFP_ATOMIC,
6029                                 AUDIT_ANOM_PROMISCUOUS,
6030                                 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
6031                                 dev->name, (dev->flags & IFF_PROMISC),
6032                                 (old_flags & IFF_PROMISC),
6033                                 from_kuid(&init_user_ns, audit_get_loginuid(current)),
6034                                 from_kuid(&init_user_ns, uid),
6035                                 from_kgid(&init_user_ns, gid),
6036                                 audit_get_sessionid(current));
6037                 }
6038 
6039                 dev_change_rx_flags(dev, IFF_PROMISC);
6040         }
6041         if (notify)
6042                 __dev_notify_flags(dev, old_flags, IFF_PROMISC);
6043         return 0;
6044 }
6045 
6046 /**
6047  *      dev_set_promiscuity     - update promiscuity count on a device
6048  *      @dev: device
6049  *      @inc: modifier
6050  *
6051  *      Add or remove promiscuity from a device. While the count in the device
6052  *      remains above zero the interface remains promiscuous. Once it hits zero
6053  *      the device reverts back to normal filtering operation. A negative inc
6054  *      value is used to drop promiscuity on the device.
6055  *      Return 0 if successful or a negative errno code on error.
6056  */
6057 int dev_set_promiscuity(struct net_device *dev, int inc)
6058 {
6059         unsigned int old_flags = dev->flags;
6060         int err;
6061 
6062         err = __dev_set_promiscuity(dev, inc, true);
6063         if (err < 0)
6064                 return err;
6065         if (dev->flags != old_flags)
6066                 dev_set_rx_mode(dev);
6067         return err;
6068 }
6069 EXPORT_SYMBOL(dev_set_promiscuity);
6070 
6071 static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify)
6072 {
6073         unsigned int old_flags = dev->flags, old_gflags = dev->gflags;
6074 
6075         ASSERT_RTNL();
6076 
6077         dev->flags |= IFF_ALLMULTI;
6078         dev->allmulti += inc;
6079         if (dev->allmulti == 0) {
6080                 /*
6081                  * Avoid overflow.
6082                  * If inc causes overflow, untouch allmulti and return error.
6083                  */
6084                 if (inc < 0)
6085                         dev->flags &= ~IFF_ALLMULTI;
6086                 else {
6087                         dev->allmulti -= inc;
6088                         pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n",
6089                                 dev->name);
6090                         return -EOVERFLOW;
6091                 }
6092         }
6093         if (dev->flags ^ old_flags) {
6094                 dev_change_rx_flags(dev, IFF_ALLMULTI);
6095                 dev_set_rx_mode(dev);
6096                 if (notify)
6097                         __dev_notify_flags(dev, old_flags,
6098                                            dev->gflags ^ old_gflags);
6099         }
6100         return 0;
6101 }
6102 
6103 /**
6104  *      dev_set_allmulti        - update allmulti count on a device
6105  *      @dev: device
6106  *      @inc: modifier
6107  *
6108  *      Add or remove reception of all multicast frames to a device. While the
6109  *      count in the device remains above zero the interface remains listening
6110  *      to all interfaces. Once it hits zero the device reverts back to normal
6111  *      filtering operation. A negative @inc value is used to drop the counter
6112  *      when releasing a resource needing all multicasts.
6113  *      Return 0 if successful or a negative errno code on error.
6114  */
6115 
6116 int dev_set_allmulti(struct net_device *dev, int inc)
6117 {
6118         return __dev_set_allmulti(dev, inc, true);
6119 }
6120 EXPORT_SYMBOL(dev_set_allmulti);
6121 
6122 /*
6123  *      Upload unicast and multicast address lists to device and
6124  *      configure RX filtering. When the device doesn't support unicast
6125  *      filtering it is put in promiscuous mode while unicast addresses
6126  *      are present.
6127  */
6128 void __dev_set_rx_mode(struct net_device *dev)
6129 {
6130         const struct net_device_ops *ops = dev->netdev_ops;
6131 
6132         /* dev_open will call this function so the list will stay sane. */
6133         if (!(dev->flags&IFF_UP))
6134                 return;
6135 
6136         if (!netif_device_present(dev))
6137                 return;
6138 
6139         if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
6140                 /* Unicast addresses changes may only happen under the rtnl,
6141                  * therefore calling __dev_set_promiscuity here is safe.
6142                  */
6143                 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
6144                         __dev_set_promiscuity(dev, 1, false);
6145                         dev->uc_promisc = true;
6146                 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
6147                         __dev_set_promiscuity(dev, -1, false);
6148                         dev->uc_promisc = false;
6149                 }
6150         }
6151 
6152         if (ops->ndo_set_rx_mode)
6153                 ops->ndo_set_rx_mode(dev);
6154 }
6155 
6156 void dev_set_rx_mode(struct net_device *dev)
6157 {
6158         netif_addr_lock_bh(dev);
6159         __dev_set_rx_mode(dev);
6160         netif_addr_unlock_bh(dev);
6161 }
6162 
6163 /**
6164  *      dev_get_flags - get flags reported to userspace
6165  *      @dev: device
6166  *
6167  *      Get the combination of flag bits exported through APIs to userspace.
6168  */
6169 unsigned int dev_get_flags(const struct net_device *dev)
6170 {
6171         unsigned int flags;
6172 
6173         flags = (dev->flags & ~(IFF_PROMISC |
6174                                 IFF_ALLMULTI |
6175                                 IFF_RUNNING |
6176                                 IFF_LOWER_UP |
6177                                 IFF_DORMANT)) |
6178                 (dev->gflags & (IFF_PROMISC |
6179                                 IFF_ALLMULTI));
6180 
6181         if (netif_running(dev)) {
6182                 if (netif_oper_up(dev))
6183                         flags |= IFF_RUNNING;
6184                 if (netif_carrier_ok(dev))
6185                         flags |= IFF_LOWER_UP;
6186                 if (netif_dormant(dev))
6187                         flags |= IFF_DORMANT;
6188         }
6189 
6190         return flags;
6191 }
6192 EXPORT_SYMBOL(dev_get_flags);
6193 
6194 int __dev_change_flags(struct net_device *dev, unsigned int flags)
6195 {
6196         unsigned int old_flags = dev->flags;
6197         int ret;
6198 
6199         ASSERT_RTNL();
6200 
6201         /*
6202          *      Set the flags on our device.
6203          */
6204 
6205         dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
6206                                IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
6207                                IFF_AUTOMEDIA)) |
6208                      (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
6209                                     IFF_ALLMULTI));
6210 
6211         /*
6212          *      Load in the correct multicast list now the flags have changed.
6213          */
6214 
6215         if ((old_flags ^ flags) & IFF_MULTICAST)
6216                 dev_change_rx_flags(dev, IFF_MULTICAST);
6217 
6218         dev_set_rx_mode(dev);
6219 
6220         /*
6221          *      Have we downed the interface. We handle IFF_UP ourselves
6222          *      according to user attempts to set it, rather than blindly
6223          *      setting it.
6224          */
6225 
6226         ret = 0;
6227         if ((old_flags ^ flags) & IFF_UP)
6228                 ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev);
6229 
6230         if ((flags ^ dev->gflags) & IFF_PROMISC) {
6231                 int inc = (flags & IFF_PROMISC) ? 1 : -1;
6232                 unsigned int old_flags = dev->flags;
6233 
6234                 dev->gflags ^= IFF_PROMISC;
6235 
6236                 if (__dev_set_promiscuity(dev, inc, false) >= 0)
6237                         if (dev->flags != old_flags)
6238                                 dev_set_rx_mode(dev);
6239         }
6240 
6241         /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
6242            is important. Some (broken) drivers set IFF_PROMISC, when
6243            IFF_ALLMULTI is requested not asking us and not reporting.
6244          */
6245         if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
6246                 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
6247 
6248                 dev->gflags ^= IFF_ALLMULTI;
6249                 __dev_set_allmulti(dev, inc, false);
6250         }
6251 
6252         return ret;
6253 }
6254 
6255 void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
6256                         unsigned int gchanges)
6257 {
6258         unsigned int changes = dev->flags ^ old_flags;
6259 
6260         if (gchanges)
6261                 rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC);
6262 
6263         if (changes & IFF_UP) {
6264                 if (dev->flags & IFF_UP)
6265                         call_netdevice_notifiers(NETDEV_UP, dev);
6266                 else
6267                         call_netdevice_notifiers(NETDEV_DOWN, dev);
6268         }
6269 
6270         if (dev->flags & IFF_UP &&
6271             (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) {
6272                 struct netdev_notifier_change_info change_info;
6273 
6274                 change_info.flags_changed = changes;
6275                 call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
6276                                               &change_info.info);
6277         }
6278 }
6279 
6280 /**
6281  *      dev_change_flags - change device settings
6282  *      @dev: device
6283  *      @flags: device state flags
6284  *
6285  *      Change settings on device based state flags. The flags are
6286  *      in the userspace exported format.
6287  */
6288 int dev_change_flags(struct net_device *dev, unsigned int flags)
6289 {
6290         int ret;
6291         unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags;
6292 
6293         ret = __dev_change_flags(dev, flags);
6294         if (ret < 0)
6295                 return ret;
6296 
6297         changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags);
6298         __dev_notify_flags(dev, old_flags, changes);
6299         return ret;
6300 }
6301 EXPORT_SYMBOL(dev_change_flags);
6302 
6303 static int __dev_set_mtu(struct net_device *dev, int new_mtu)
6304 {
6305         const struct net_device_ops *ops = dev->netdev_ops;
6306 
6307         if (ops->ndo_change_mtu)
6308                 return ops->ndo_change_mtu(dev, new_mtu);
6309 
6310         dev->mtu = new_mtu;
6311         return 0;
6312 }
6313 
6314 /**
6315  *      dev_set_mtu - Change maximum transfer unit
6316  *      @dev: device
6317  *      @new_mtu: new transfer unit
6318  *
6319  *      Change the maximum transfer size of the network device.
6320  */
6321 int dev_set_mtu(struct net_device *dev, int new_mtu)
6322 {
6323         int err, orig_mtu;
6324 
6325         if (new_mtu == dev->mtu)
6326                 return 0;
6327 
6328         /*      MTU must be positive.    */
6329         if (new_mtu < 0)
6330                 return -EINVAL;
6331 
6332         if (!netif_device_present(dev))
6333                 return -ENODEV;
6334 
6335         err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev);
6336         err = notifier_to_errno(err);
6337         if (err)
6338                 return err;
6339 
6340         orig_mtu = dev->mtu;
6341         err = __dev_set_mtu(dev, new_mtu);
6342 
6343         if (!err) {
6344                 err = call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
6345                 err = notifier_to_errno(err);
6346                 if (err) {
6347                         /* setting mtu back and notifying everyone again,
6348                          * so that they have a chance to revert changes.
6349                          */
6350                         __dev_set_mtu(dev, orig_mtu);
6351                         call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
6352                 }
6353         }
6354         return err;
6355 }
6356 EXPORT_SYMBOL(dev_set_mtu);
6357 
6358 /**
6359  *      dev_set_group - Change group this device belongs to
6360  *      @dev: device
6361  *      @new_group: group this device should belong to
6362  */
6363 void dev_set_group(struct net_device *dev, int new_group)
6364 {
6365         dev->group = new_group;
6366 }
6367 EXPORT_SYMBOL(dev_set_group);
6368 
6369 /**
6370  *      dev_set_mac_address - Change Media Access Control Address
6371  *      @dev: device
6372  *      @sa: new address
6373  *
6374  *      Change the hardware (MAC) address of the device
6375  */
6376 int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
6377 {
6378         const struct net_device_ops *ops = dev->netdev_ops;
6379         int err;
6380 
6381         if (!ops->ndo_set_mac_address)
6382                 return -EOPNOTSUPP;
6383         if (sa->sa_family != dev->type)
6384                 return -EINVAL;
6385         if (!netif_device_present(dev))
6386                 return -ENODEV;
6387         err = ops->ndo_set_mac_address(dev, sa);
6388         if (err)
6389                 return err;
6390         dev->addr_assign_type = NET_ADDR_SET;
6391         call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
6392         add_device_randomness(dev->dev_addr, dev->addr_len);
6393         return 0;
6394 }
6395 EXPORT_SYMBOL(dev_set_mac_address);
6396 
6397 /**
6398  *      dev_change_carrier - Change device carrier
6399  *      @dev: device
6400  *      @new_carrier: new value
6401  *
6402  *      Change device carrier
6403  */
6404 int dev_change_carrier(struct net_device *dev, bool new_carrier)
6405 {
6406         const struct net_device_ops *ops = dev->netdev_ops;
6407 
6408         if (!ops->ndo_change_carrier)
6409                 return -EOPNOTSUPP;
6410         if (!netif_device_present(dev))
6411                 return -ENODEV;
6412         return ops->ndo_change_carrier(dev, new_carrier);
6413 }
6414 EXPORT_SYMBOL(dev_change_carrier);
6415 
6416 /**
6417  *      dev_get_phys_port_id - Get device physical port ID
6418  *      @dev: device
6419  *      @ppid: port ID
6420  *
6421  *      Get device physical port ID
6422  */
6423 int dev_get_phys_port_id(struct net_device *dev,
6424                          struct netdev_phys_item_id *ppid)
6425 {
6426         const struct net_device_ops *ops = dev->netdev_ops;
6427 
6428         if (!ops->ndo_get_phys_port_id)
6429                 return -EOPNOTSUPP;
6430         return ops->ndo_get_phys_port_id(dev, ppid);
6431 }
6432 EXPORT_SYMBOL(dev_get_phys_port_id);
6433 
6434 /**
6435  *      dev_get_phys_port_name - Get device physical port name
6436  *      @dev: device
6437  *      @name: port name
6438  *
6439  *      Get device physical port name
6440  */
6441 int dev_get_phys_port_name(struct net_device *dev,
6442                            char *name, size_t len)
6443 {
6444         const struct net_device_ops *ops = dev->netdev_ops;
6445 
6446         if (!ops->ndo_get_phys_port_name)
6447                 return -EOPNOTSUPP;
6448         return ops->ndo_get_phys_port_name(dev, name, len);
6449 }
6450 EXPORT_SYMBOL(dev_get_phys_port_name);
6451 
6452 /**
6453  *      dev_change_proto_down - update protocol port state information
6454  *      @dev: device
6455  *      @proto_down: new value
6456  *
6457  *      This info can be used by switch drivers to set the phys state of the
6458  *      port.
6459  */
6460 int dev_change_proto_down(struct net_device *dev, bool proto_down)
6461 {
6462         const struct net_device_ops *ops = dev->netdev_ops;
6463 
6464         if (!ops->ndo_change_proto_down)
6465                 return -EOPNOTSUPP;
6466         if (!netif_device_present(dev))
6467                 return -ENODEV;
6468         return ops->ndo_change_proto_down(dev, proto_down);
6469 }
6470 EXPORT_SYMBOL(dev_change_proto_down);
6471 
6472 /**
6473  *      dev_new_index   -       allocate an ifindex
6474  *      @net: the applicable net namespace
6475  *
6476  *      Returns a suitable unique value for a new device interface
6477  *      number.  The caller must hold the rtnl semaphore or the
6478  *      dev_base_lock to be sure it remains unique.
6479  */
6480 static int dev_new_index(struct net *net)
6481 {
6482         int ifindex = net->ifindex;
6483         for (;;) {
6484                 if (++ifindex <= 0)
6485                         ifindex = 1;
6486                 if (!__dev_get_by_index(net, ifindex))
6487                         return net->ifindex = ifindex;
6488         }
6489 }
6490 
6491 /* Delayed registration/unregisteration */
6492 static LIST_HEAD(net_todo_list);
6493 DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq);
6494 
6495 static void net_set_todo(struct net_device *dev)
6496 {
6497         list_add_tail(&dev->todo_list, &net_todo_list);
6498         dev_net(dev)->dev_unreg_count++;
6499 }
6500 
6501 static void rollback_registered_many(struct list_head *head)
6502 {
6503         struct net_device *dev, *tmp;
6504         LIST_HEAD(close_head);
6505 
6506         BUG_ON(dev_boot_phase);
6507         ASSERT_RTNL();
6508 
6509         list_for_each_entry_safe(dev, tmp, head, unreg_list) {
6510                 /* Some devices call without registering
6511                  * for initialization unwind. Remove those
6512                  * devices and proceed with the remaining.
6513                  */
6514                 if (dev->reg_state == NETREG_UNINITIALIZED) {
6515                         pr_debug("unregister_netdevice: device %s/%p never was registered\n",
6516                                  dev->name, dev);
6517 
6518                         WARN_ON(1);
6519                         list_del(&dev->unreg_list);
6520                         continue;
6521                 }
6522                 dev->dismantle = true;
6523                 BUG_ON(dev->reg_state != NETREG_REGISTERED);
6524         }
6525 
6526         /* If device is running, close it first. */
6527         list_for_each_entry(dev, head, unreg_list)
6528                 list_add_tail(&dev->close_list, &close_head);
6529         dev_close_many(&close_head, true);
6530 
6531         list_for_each_entry(dev, head, unreg_list) {
6532                 /* And unlink it from device chain. */
6533                 unlist_netdevice(dev);
6534 
6535                 dev->reg_state = NETREG_UNREGISTERING;
6536                 on_each_cpu(flush_backlog, dev, 1);
6537         }
6538 
6539         synchronize_net();
6540 
6541         list_for_each_entry(dev, head, unreg_list) {
6542                 struct sk_buff *skb = NULL;
6543 
6544                 /* Shutdown queueing discipline. */
6545                 dev_shutdown(dev);
6546 
6547 
6548                 /* Notify protocols, that we are about to destroy
6549                    this device. They should clean all the things.
6550                 */
6551                 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
6552 
6553                 if (!dev->rtnl_link_ops ||
6554                     dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
6555                         skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U,
6556                                                      GFP_KERNEL);
6557 
6558                 /*
6559                  *      Flush the unicast and multicast chains
6560                  */
6561                 dev_uc_flush(dev);
6562                 dev_mc_flush(dev);
6563 
6564                 if (dev->netdev_ops->ndo_uninit)
6565                         dev->netdev_ops->ndo_uninit(dev);
6566 
6567                 if (skb)
6568                         rtmsg_ifinfo_send(skb, dev, GFP_KERNEL);
6569 
6570                 /* Notifier chain MUST detach us all upper devices. */
6571                 WARN_ON(netdev_has_any_upper_dev(dev));
6572 
6573                 /* Remove entries from kobject tree */
6574                 netdev_unregister_kobject(dev);
6575 #ifdef CONFIG_XPS
6576                 /* Remove XPS queueing entries */
6577                 netif_reset_xps_queues_gt(dev, 0);
6578 #endif
6579         }
6580 
6581         synchronize_net();
6582 
6583         list_for_each_entry(dev, head, unreg_list)
6584                 dev_put(dev);
6585 }
6586 
6587 static void rollback_registered(struct net_device *dev)
6588 {
6589         LIST_HEAD(single);
6590 
6591         list_add(&dev->unreg_list, &single);
6592         rollback_registered_many(&single);
6593         list_del(&single);
6594 }
6595 
6596 static netdev_features_t netdev_sync_upper_features(struct net_device *lower,
6597         struct net_device *upper, netdev_features_t features)
6598 {
6599         netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
6600         netdev_features_t feature;
6601         int feature_bit;
6602 
6603         for_each_netdev_feature(&upper_disables, feature_bit) {
6604                 feature = __NETIF_F_BIT(feature_bit);
6605                 if (!(upper->wanted_features & feature)
6606                     && (features & feature)) {
6607                         netdev_dbg(lower, "Dropping feature %pNF, upper dev %s has it off.\n",
6608                                    &feature, upper->name);
6609                         features &= ~feature;
6610                 }
6611         }
6612 
6613         return features;
6614 }
6615 
6616 static void netdev_sync_lower_features(struct net_device *upper,
6617         struct net_device *lower, netdev_features_t features)
6618 {
6619         netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
6620         netdev_features_t feature;
6621         int feature_bit;
6622 
6623         for_each_netdev_feature(&upper_disables, feature_bit) {
6624                 feature = __NETIF_F_BIT(feature_bit);
6625                 if (!(features & feature) && (lower->features & feature)) {
6626                         netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n",
6627                                    &feature, lower->name);
6628                         lower->wanted_features &= ~feature;
6629                         netdev_update_features(lower);
6630 
6631                         if (unlikely(lower->features & feature))
6632                                 netdev_WARN(upper, "failed to disable %pNF on %s!\n",
6633                                             &feature, lower->name);
6634                 }
6635         }
6636 }
6637 
6638 static netdev_features_t netdev_fix_features(struct net_device *dev,
6639         netdev_features_t features)
6640 {
6641         /* Fix illegal checksum combinations */
6642         if ((features & NETIF_F_HW_CSUM) &&
6643             (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
6644                 netdev_warn(dev, "mixed HW and IP checksum settings.\n");
6645                 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
6646         }
6647 
6648         /* TSO requires that SG is present as well. */
6649         if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
6650                 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
6651                 features &= ~NETIF_F_ALL_TSO;
6652         }
6653 
6654         if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) &&
6655                                         !(features & NETIF_F_IP_CSUM)) {
6656                 netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n");
6657                 features &= ~NETIF_F_TSO;
6658                 features &= ~NETIF_F_TSO_ECN;
6659         }
6660 
6661         if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) &&
6662                                          !(features & NETIF_F_IPV6_CSUM)) {
6663                 netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n");
6664                 features &= ~NETIF_F_TSO6;
6665         }
6666 
6667         /* TSO ECN requires that TSO is present as well. */
6668         if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
6669                 features &= ~NETIF_F_TSO_ECN;
6670 
6671         /* Software GSO depends on SG. */
6672         if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
6673                 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
6674                 features &= ~NETIF_F_GSO;
6675         }
6676 
6677         /* UFO needs SG and checksumming */
6678         if (features & NETIF_F_UFO) {
6679                 /* maybe split UFO into V4 and V6? */
6680                 if (!(features & NETIF_F_HW_CSUM) &&
6681                     ((features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) !=
6682                      (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM))) {
6683                         netdev_dbg(dev,
6684                                 "Dropping NETIF_F_UFO since no checksum offload features.\n");
6685                         features &= ~NETIF_F_UFO;
6686                 }
6687 
6688                 if (!(features & NETIF_F_SG)) {
6689                         netdev_dbg(dev,
6690                                 "Dropping NETIF_F_UFO since no NETIF_F_SG feature.\n");
6691                         features &= ~NETIF_F_UFO;
6692                 }
6693         }
6694 
6695 #ifdef CONFIG_NET_RX_BUSY_POLL
6696         if (dev->netdev_ops->ndo_busy_poll)
6697                 features |= NETIF_F_BUSY_POLL;
6698         else
6699 #endif
6700                 features &= ~NETIF_F_BUSY_POLL;
6701 
6702         return features;
6703 }
6704 
6705 int __netdev_update_features(struct net_device *dev)
6706 {
6707         struct net_device *upper, *lower;
6708         netdev_features_t features;
6709         struct list_head *iter;
6710         int err = -1;
6711 
6712         ASSERT_RTNL();
6713 
6714         features = netdev_get_wanted_features(dev);
6715 
6716         if (dev->netdev_ops->ndo_fix_features)
6717                 features = dev->netdev_ops->ndo_fix_features(dev, features);
6718 
6719         /* driver might be less strict about feature dependencies */
6720         features = netdev_fix_features(dev, features);
6721 
6722         /* some features can't be enabled if they're off an an upper device */
6723         netdev_for_each_upper_dev_rcu(dev, upper, iter)
6724                 features = netdev_sync_upper_features(dev, upper, features);
6725 
6726         if (dev->features == features)
6727                 goto sync_lower;
6728 
6729         netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
6730                 &dev->features, &features);
6731 
6732         if (dev->netdev_ops->ndo_set_features)
6733                 err = dev->netdev_ops->ndo_set_features(dev, features);
6734         else
6735                 err = 0;
6736 
6737         if (unlikely(err < 0)) {
6738                 netdev_err(dev,
6739                         "set_features() failed (%d); wanted %pNF, left %pNF\n",
6740                         err, &features, &dev->features);
6741                 /* return non-0 since some features might have changed and
6742                  * it's better to fire a spurious notification than miss it
6743                  */
6744                 return -1;
6745         }
6746 
6747 sync_lower:
6748         /* some features must be disabled on lower devices when disabled
6749          * on an upper device (think: bonding master or bridge)
6750          */
6751         netdev_for_each_lower_dev(dev, lower, iter)
6752                 netdev_sync_lower_features(dev, lower, features);
6753 
6754         if (!err)
6755                 dev->features = features;
6756 
6757         return err < 0 ? 0 : 1;
6758 }
6759 
6760 /**
6761  *      netdev_update_features - recalculate device features
6762  *      @dev: the device to check
6763  *
6764  *      Recalculate dev->features set and send notifications if it
6765  *      has changed. Should be called after driver or hardware dependent
6766  *      conditions might have changed that influence the features.
6767  */
6768 void netdev_update_features(struct net_device *dev)
6769 {
6770         if (__netdev_update_features(dev))
6771                 netdev_features_change(dev);
6772 }
6773 EXPORT_SYMBOL(netdev_update_features);
6774 
6775 /**
6776  *      netdev_change_features - recalculate device features
6777  *      @dev: the device to check
6778  *
6779  *      Recalculate dev->features set and send notifications even
6780  *      if they have not changed. Should be called instead of
6781  *      netdev_update_features() if also dev->vlan_features might
6782  *      have changed to allow the changes to be propagated to stacked
6783  *      VLAN devices.
6784  */
6785 void netdev_change_features(struct net_device *dev)
6786 {
6787         __netdev_update_features(dev);
6788         netdev_features_change(dev);
6789 }
6790 EXPORT_SYMBOL(netdev_change_features);
6791 
6792 /**
6793  *      netif_stacked_transfer_operstate -      transfer operstate
6794  *      @rootdev: the root or lower level device to transfer state from
6795  *      @dev: the device to transfer operstate to
6796  *
6797  *      Transfer operational state from root to device. This is normally
6798  *      called when a stacking relationship exists between the root
6799  *      device and the device(a leaf device).
6800  */
6801 void netif_stacked_transfer_operstate(const struct net_device *rootdev,
6802                                         struct net_device *dev)
6803 {
6804         if (rootdev->operstate == IF_OPER_DORMANT)
6805                 netif_dormant_on(dev);
6806         else
6807                 netif_dormant_off(dev);
6808 
6809         if (netif_carrier_ok(rootdev)) {
6810                 if (!netif_carrier_ok(dev))
6811                         netif_carrier_on(dev);
6812         } else {
6813                 if (netif_carrier_ok(dev))
6814                         netif_carrier_off(dev);
6815         }
6816 }
6817 EXPORT_SYMBOL(netif_stacked_transfer_operstate);
6818 
6819 #ifdef CONFIG_SYSFS
6820 static int netif_alloc_rx_queues(struct net_device *dev)
6821 {
6822         unsigned int i, count = dev->num_rx_queues;
6823         struct netdev_rx_queue *rx;
6824         size_t sz = count * sizeof(*rx);
6825 
6826         BUG_ON(count < 1);
6827 
6828         rx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
6829         if (!rx) {
6830                 rx = vzalloc(sz);
6831                 if (!rx)
6832                         return -ENOMEM;
6833         }
6834         dev->_rx = rx;
6835 
6836         for (i = 0; i < count; i++)
6837                 rx[i].dev = dev;
6838         return 0;
6839 }
6840 #endif
6841 
6842 static void netdev_init_one_queue(struct net_device *dev,
6843                                   struct netdev_queue *queue, void *_unused)
6844 {
6845         /* Initialize queue lock */
6846         spin_lock_init(&queue->_xmit_lock);
6847         netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
6848         queue->xmit_lock_owner = -1;
6849         netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
6850         queue->dev = dev;
6851 #ifdef CONFIG_BQL
6852         dql_init(&queue->dql, HZ);
6853 #endif
6854 }
6855 
6856 static void netif_free_tx_queues(struct net_device *dev)
6857 {
6858         kvfree(dev->_tx);
6859 }
6860 
6861 static int netif_alloc_netdev_queues(struct net_device *dev)
6862 {
6863         unsigned int count = dev->num_tx_queues;
6864         struct netdev_queue *tx;
6865         size_t sz = count * sizeof(*tx);
6866 
6867         if (count < 1 || count > 0xffff)
6868                 return -EINVAL;
6869 
6870         tx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
6871         if (!tx) {
6872                 tx = vzalloc(sz);
6873                 if (!tx)
6874                         return -ENOMEM;
6875         }
6876         dev->_tx = tx;
6877 
6878         netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
6879         spin_lock_init(&dev->tx_global_lock);
6880 
6881         return 0;
6882 }
6883 
6884 void netif_tx_stop_all_queues(struct net_device *dev)
6885 {
6886         unsigned int i;
6887 
6888         for (i = 0; i < dev->num_tx_queues; i++) {
6889                 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
6890                 netif_tx_stop_queue(txq);
6891         }
6892 }
6893 EXPORT_SYMBOL(netif_tx_stop_all_queues);
6894 
6895 /**
6896  *      register_netdevice      - register a network device
6897  *      @dev: device to register
6898  *
6899  *      Take a completed network device structure and add it to the kernel
6900  *      interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
6901  *      chain. 0 is returned on success. A negative errno code is returned
6902  *      on a failure to set up the device, or if the name is a duplicate.
6903  *
6904  *      Callers must hold the rtnl semaphore. You may want
6905  *      register_netdev() instead of this.
6906  *
6907  *      BUGS:
6908  *      The locking appears insufficient to guarantee two parallel registers
6909  *      will not get the same name.
6910  */
6911 
6912 int register_netdevice(struct net_device *dev)
6913 {
6914         int ret;
6915         struct net *net = dev_net(dev);
6916 
6917         BUG_ON(dev_boot_phase);
6918         ASSERT_RTNL();
6919 
6920         might_sleep();
6921 
6922         /* When net_device's are persistent, this will be fatal. */
6923         BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
6924         BUG_ON(!net);
6925 
6926         spin_lock_init(&dev->addr_list_lock);
6927         netdev_set_addr_lockdep_class(dev);
6928 
6929         ret = dev_get_valid_name(net, dev, dev->name);
6930         if (ret < 0)
6931                 goto out;
6932 
6933         /* Init, if this function is available */
6934         if (dev->netdev_ops->ndo_init) {
6935                 ret = dev->netdev_ops->ndo_init(dev);
6936                 if (ret) {
6937                         if (ret > 0)
6938                                 ret = -EIO;
6939                         goto out;
6940                 }
6941         }
6942 
6943         if (((dev->hw_features | dev->features) &
6944              NETIF_F_HW_VLAN_CTAG_FILTER) &&
6945             (!dev->netdev_ops->ndo_vlan_rx_add_vid ||
6946              !dev->netdev_ops->ndo_vlan_rx_kill_vid)) {
6947                 netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n");
6948                 ret = -EINVAL;
6949                 goto err_uninit;
6950         }
6951 
6952         ret = -EBUSY;
6953         if (!dev->ifindex)
6954                 dev->ifindex = dev_new_index(net);
6955         else if (__dev_get_by_index(net, dev->ifindex))
6956                 goto err_uninit;
6957 
6958         /* Transfer changeable features to wanted_features and enable
6959          * software offloads (GSO and GRO).
6960          */
6961         dev->hw_features |= NETIF_F_SOFT_FEATURES;
6962         dev->features |= NETIF_F_SOFT_FEATURES;
6963         dev->wanted_features = dev->features & dev->hw_features;
6964 
6965         if (!(dev->flags & IFF_LOOPBACK)) {
6966                 dev->hw_features |= NETIF_F_NOCACHE_COPY;
6967         }
6968 
6969         /* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
6970          */
6971         dev->vlan_features |= NETIF_F_HIGHDMA;
6972 
6973         /* Make NETIF_F_SG inheritable to tunnel devices.
6974          */
6975         dev->hw_enc_features |= NETIF_F_SG;
6976 
6977         /* Make NETIF_F_SG inheritable to MPLS.
6978          */
6979         dev->mpls_features |= NETIF_F_SG;
6980 
6981         ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
6982         ret = notifier_to_errno(ret);
6983         if (ret)
6984                 goto err_uninit;
6985 
6986         ret = netdev_register_kobject(dev);
6987         if (ret)
6988                 goto err_uninit;
6989         dev->reg_state = NETREG_REGISTERED;
6990 
6991         __netdev_update_features(dev);
6992 
6993         /*
6994          *      Default initial state at registry is that the
6995          *      device is present.
6996          */
6997 
6998         set_bit(__LINK_STATE_PRESENT, &dev->state);
6999 
7000         linkwatch_init_dev(dev);
7001 
7002         dev_init_scheduler(dev);
7003         dev_hold(dev);
7004         list_netdevice(dev);
7005         add_device_randomness(dev->dev_addr, dev->addr_len);
7006 
7007         /* If the device has permanent device address, driver should
7008          * set dev_addr and also addr_assign_type should be set to
7009          * NET_ADDR_PERM (default value).
7010          */
7011         if (dev->addr_assign_type == NET_ADDR_PERM)
7012                 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
7013 
7014         /* Notify protocols, that a new device appeared. */
7015         ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
7016         ret = notifier_to_errno(ret);
7017         if (ret) {
7018                 rollback_registered(dev);
7019                 dev->reg_state = NETREG_UNREGISTERED;
7020         }
7021         /*
7022          *      Prevent userspace races by waiting until the network
7023          *      device is fully setup before sending notifications.
7024          */
7025         if (!dev->rtnl_link_ops ||
7026             dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
7027                 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
7028 
7029 out:
7030         return ret;
7031 
7032 err_uninit:
7033         if (dev->netdev_ops->ndo_uninit)
7034                 dev->netdev_ops->ndo_uninit(dev);
7035         goto out;
7036 }
7037 EXPORT_SYMBOL(register_netdevice);
7038 
7039 /**
7040  *      init_dummy_netdev       - init a dummy network device for NAPI
7041  *      @dev: device to init
7042  *
7043  *      This takes a network device structure and initialize the minimum
7044  *      amount of fields so it can be used to schedule NAPI polls without
7045  *      registering a full blown interface. This is to be used by drivers
7046  *      that need to tie several hardware interfaces to a single NAPI
7047  *      poll scheduler due to HW limitations.
7048  */
7049 int init_dummy_netdev(struct net_device *dev)
7050 {
7051         /* Clear everything. Note we don't initialize spinlocks
7052          * are they aren't supposed to be taken by any of the
7053          * NAPI code and this dummy netdev is supposed to be
7054          * only ever used for NAPI polls
7055          */
7056         memset(dev, 0, sizeof(struct net_device));
7057 
7058         /* make sure we BUG if trying to hit standard
7059          * register/unregister code path
7060          */
7061         dev->reg_state = NETREG_DUMMY;
7062 
7063         /* NAPI wants this */
7064         INIT_LIST_HEAD(&dev->napi_list);
7065 
7066         /* a dummy interface is started by default */
7067         set_bit(__LINK_STATE_PRESENT, &dev->state);
7068         set_bit(__LINK_STATE_START, &dev->state);
7069 
7070         /* Note : We dont allocate pcpu_refcnt for dummy devices,
7071          * because users of this 'device' dont need to change
7072          * its refcount.
7073          */
7074 
7075         return 0;
7076 }
7077 EXPORT_SYMBOL_GPL(init_dummy_netdev);
7078 
7079 
7080 /**
7081  *      register_netdev - register a network device
7082  *      @dev: device to register
7083  *
7084  *      Take a completed network device structure and add it to the kernel
7085  *      interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
7086  *      chain. 0 is returned on success. A negative errno code is returned
7087  *      on a failure to set up the device, or if the name is a duplicate.
7088  *
7089  *      This is a wrapper around register_netdevice that takes the rtnl semaphore
7090  *      and expands the device name if you passed a format string to
7091  *      alloc_netdev.
7092  */
7093 int register_netdev(struct net_device *dev)
7094 {
7095         int err;
7096 
7097         rtnl_lock();
7098         err = register_netdevice(dev);
7099         rtnl_unlock();
7100         return err;
7101 }
7102 EXPORT_SYMBOL(register_netdev);
7103 
7104 int netdev_refcnt_read(const struct net_device *dev)
7105 {
7106         int i, refcnt = 0;
7107 
7108         for_each_possible_cpu(i)
7109                 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
7110         return refcnt;
7111 }
7112 EXPORT_SYMBOL(netdev_refcnt_read);
7113 
7114 /**
7115  * netdev_wait_allrefs - wait until all references are gone.
7116  * @dev: target net_device
7117  *
7118  * This is called when unregistering network devices.
7119  *
7120  * Any protocol or device that holds a reference should register
7121  * for netdevice notification, and cleanup and put back the
7122  * reference if they receive an UNREGISTER event.
7123  * We can get stuck here if buggy protocols don't correctly
7124  * call dev_put.
7125  */
7126 static void netdev_wait_allrefs(struct net_device *dev)
7127 {
7128         unsigned long rebroadcast_time, warning_time;
7129         int refcnt;
7130 
7131         linkwatch_forget_dev(dev);
7132 
7133         rebroadcast_time = warning_time = jiffies;
7134         refcnt = netdev_refcnt_read(dev);
7135 
7136         while (refcnt != 0) {
7137                 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
7138                         rtnl_lock();
7139 
7140                         /* Rebroadcast unregister notification */
7141                         call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
7142 
7143                         __rtnl_unlock();
7144                         rcu_barrier();
7145                         rtnl_lock();
7146 
7147                         call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
7148                         if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
7149                                      &dev->state)) {
7150                                 /* We must not have linkwatch events
7151                                  * pending on unregister. If this
7152                                  * happens, we simply run the queue
7153                                  * unscheduled, resulting in a noop
7154                                  * for this device.
7155                                  */
7156                                 linkwatch_run_queue();
7157                         }
7158 
7159                         __rtnl_unlock();
7160 
7161                         rebroadcast_time = jiffies;
7162                 }
7163 
7164                 msleep(250);
7165 
7166                 refcnt = netdev_refcnt_read(dev);
7167 
7168                 if (time_after(jiffies, warning_time + 10 * HZ)) {
7169                         pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
7170                                  dev->name, refcnt);
7171                         warning_time = jiffies;
7172                 }
7173         }
7174 }
7175 
7176 /* The sequence is:
7177  *
7178  *      rtnl_lock();
7179  *      ...
7180  *      register_netdevice(x1);
7181  *      register_netdevice(x2);
7182  *      ...
7183  *      unregister_netdevice(y1);
7184  *      unregister_netdevice(y2);
7185  *      ...
7186  *      rtnl_unlock();
7187  *      free_netdev(y1);
7188  *      free_netdev(y2);
7189  *
7190  * We are invoked by rtnl_unlock().
7191  * This allows us to deal with problems:
7192  * 1) We can delete sysfs objects which invoke hotplug
7193  *    without deadlocking with linkwatch via keventd.
7194  * 2) Since we run with the RTNL semaphore not held, we can sleep
7195  *    safely in order to wait for the netdev refcnt to drop to zero.
7196  *
7197  * We must not return until all unregister events added during
7198  * the interval the lock was held have been completed.
7199  */
7200 void netdev_run_todo(void)
7201 {
7202         struct list_head list;
7203 
7204         /* Snapshot list, allow later requests */
7205         list_replace_init(&net_todo_list, &list);
7206 
7207         __rtnl_unlock();
7208 
7209 
7210         /* Wait for rcu callbacks to finish before next phase */
7211         if (!list_empty(&list))
7212                 rcu_barrier();
7213 
7214         while (!list_empty(&list)) {
7215                 struct net_device *dev
7216                         = list_first_entry(&list, struct net_device, todo_list);
7217                 list_del(&dev->todo_list);
7218 
7219                 rtnl_lock();
7220                 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
7221                 __rtnl_unlock();
7222 
7223                 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
7224                         pr_err("network todo '%s' but state %d\n",
7225                                dev->name, dev->reg_state);
7226                         dump_stack();
7227                         continue;
7228                 }
7229 
7230                 dev->reg_state = NETREG_UNREGISTERED;
7231 
7232                 netdev_wait_allrefs(dev);
7233 
7234                 /* paranoia */
7235                 BUG_ON(netdev_refcnt_read(dev));
7236                 BUG_ON(!list_empty(&dev->ptype_all));
7237                 BUG_ON(!list_empty(&dev->ptype_specific));
7238                 WARN_ON(rcu_access_pointer(dev->ip_ptr));
7239                 WARN_ON(rcu_access_pointer(dev->ip6_ptr));
7240                 WARN_ON(dev->dn_ptr);
7241 
7242                 if (dev->destructor)
7243                         dev->destructor(dev);
7244 
7245                 /* Report a network device has been unregistered */
7246                 rtnl_lock();
7247                 dev_net(dev)->dev_unreg_count--;
7248                 __rtnl_unlock();
7249                 wake_up(&netdev_unregistering_wq);
7250 
7251                 /* Free network device */
7252                 kobject_put(&dev->dev.kobj);
7253         }
7254 }
7255 
7256 /* Convert net_device_stats to rtnl_link_stats64.  They have the same
7257  * fields in the same order, with only the type differing.
7258  */
7259 void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
7260                              const struct net_device_stats *netdev_stats)
7261 {
7262 #if BITS_PER_LONG == 64
7263         BUILD_BUG_ON(sizeof(*stats64) != sizeof(*netdev_stats));
7264         memcpy(stats64, netdev_stats, sizeof(*stats64));
7265 #else
7266         size_t i, n = sizeof(*stats64) / sizeof(u64);
7267         const unsigned long *src = (const unsigned long *)netdev_stats;
7268         u64 *dst = (u64 *)stats64;
7269 
7270         BUILD_BUG_ON(sizeof(*netdev_stats) / sizeof(unsigned long) !=
7271                      sizeof(*stats64) / sizeof(u64));
7272         for (i = 0; i < n; i++)
7273                 dst[i] = src[i];
7274 #endif
7275 }
7276 EXPORT_SYMBOL(netdev_stats_to_stats64);
7277 
7278 /**
7279  *      dev_get_stats   - get network device statistics
7280  *      @dev: device to get statistics from
7281  *      @storage: place to store stats
7282  *
7283  *      Get network statistics from device. Return @storage.
7284  *      The device driver may provide its own method by setting
7285  *      dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
7286  *      otherwise the internal statistics structure is used.
7287  */
7288 struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
7289                                         struct rtnl_link_stats64 *storage)
7290 {
7291         const struct net_device_ops *ops = dev->netdev_ops;
7292 
7293         if (ops->ndo_get_stats64) {
7294                 memset(storage, 0, sizeof(*storage));
7295                 ops->ndo_get_stats64(dev, storage);
7296         } else if (ops->ndo_get_stats) {
7297                 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
7298         } else {
7299                 netdev_stats_to_stats64(storage, &dev->stats);
7300         }
7301         storage->rx_dropped += atomic_long_read(&dev->rx_dropped);
7302         storage->tx_dropped += atomic_long_read(&dev->tx_dropped);
7303         return storage;
7304 }
7305 EXPORT_SYMBOL(dev_get_stats);
7306 
7307 struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
7308 {
7309         struct netdev_queue *queue = dev_ingress_queue(dev);
7310 
7311 #ifdef CONFIG_NET_CLS_ACT
7312         if (queue)
7313                 return queue;
7314         queue = kzalloc(sizeof(*queue), GFP_KERNEL);
7315         if (!queue)
7316                 return NULL;
7317         netdev_init_one_queue(dev, queue, NULL);
7318         RCU_INIT_POINTER(queue->qdisc, &noop_qdisc);
7319         queue->qdisc_sleeping = &noop_qdisc;
7320         rcu_assign_pointer(dev->ingress_queue, queue);
7321 #endif
7322         return queue;
7323 }
7324 
7325 static const struct ethtool_ops default_ethtool_ops;
7326 
7327 void netdev_set_default_ethtool_ops(struct net_device *dev,
7328                                     const struct ethtool_ops *ops)
7329 {
7330         if (dev->ethtool_ops == &default_ethtool_ops)
7331                 dev->ethtool_ops = ops;
7332 }
7333 EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops);
7334 
7335 void netdev_freemem(struct net_device *dev)
7336 {
7337         char *addr = (char *)dev - dev->padded;
7338 
7339         kvfree(addr);
7340 }
7341 
7342 /**
7343  *      alloc_netdev_mqs - allocate network device
7344  *      @sizeof_priv:           size of private data to allocate space for
7345  *      @name:                  device name format string
7346  *      @name_assign_type:      origin of device name
7347  *      @setup:                 callback to initialize device
7348  *      @txqs:                  the number of TX subqueues to allocate
7349  *      @rxqs:                  the number of RX subqueues to allocate
7350  *
7351  *      Allocates a struct net_device with private data area for driver use
7352  *      and performs basic initialization.  Also allocates subqueue structs
7353  *      for each queue on the device.
7354  */
7355 struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
7356                 unsigned char name_assign_type,
7357                 void (*setup)(struct net_device *),
7358                 unsigned int txqs, unsigned int rxqs)
7359 {
7360         struct net_device *dev;
7361         size_t alloc_size;
7362         struct net_device *p;
7363 
7364         BUG_ON(strlen(name) >= sizeof(dev->name));
7365 
7366         if (txqs < 1) {
7367                 pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
7368                 return NULL;
7369         }
7370 
7371 #ifdef CONFIG_SYSFS
7372         if (rxqs < 1) {
7373                 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
7374                 return NULL;
7375         }
7376 #endif
7377 
7378         alloc_size = sizeof(struct net_device);
7379         if (sizeof_priv) {
7380                 /* ensure 32-byte alignment of private area */
7381                 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
7382                 alloc_size += sizeof_priv;
7383         }
7384         /* ensure 32-byte alignment of whole construct */
7385         alloc_size += NETDEV_ALIGN - 1;
7386 
7387         p = kzalloc(alloc_size, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
7388         if (!p)
7389                 p = vzalloc(alloc_size);
7390         if (!p)
7391                 return NULL;
7392 
7393         dev = PTR_ALIGN(p, NETDEV_ALIGN);
7394         dev->padded = (char *)dev - (char *)p;
7395 
7396         dev->pcpu_refcnt = alloc_percpu(int);
7397         if (!dev->pcpu_refcnt)
7398                 goto free_dev;
7399 
7400         if (dev_addr_init(dev))
7401                 goto free_pcpu;
7402 
7403         dev_mc_init(dev);
7404         dev_uc_init(dev);
7405 
7406         dev_net_set(dev, &init_net);
7407 
7408         dev->gso_max_size = GSO_MAX_SIZE;
7409         dev->gso_max_segs = GSO_MAX_SEGS;
7410         dev->gso_min_segs = 0;
7411 
7412         INIT_LIST_HEAD(&dev->napi_list);
7413         INIT_LIST_HEAD(&dev->unreg_list);
7414         INIT_LIST_HEAD(&dev->close_list);
7415         INIT_LIST_HEAD(&dev->link_watch_list);
7416         INIT_LIST_HEAD(&dev->adj_list.upper);
7417         INIT_LIST_HEAD(&dev->adj_list.lower);
7418         INIT_LIST_HEAD(&dev->all_adj_list.upper);
7419         INIT_LIST_HEAD(&dev->all_adj_list.lower);
7420         INIT_LIST_HEAD(&dev->ptype_all);
7421         INIT_LIST_HEAD(&dev->ptype_specific);
7422         dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM;
7423         setup(dev);
7424 
7425         if (!dev->tx_queue_len) {
7426                 dev->priv_flags |= IFF_NO_QUEUE;
7427                 dev->tx_queue_len = 1;
7428         }
7429 
7430         dev->num_tx_queues = txqs;
7431         dev->real_num_tx_queues = txqs;
7432         if (netif_alloc_netdev_queues(dev))
7433                 goto free_all;
7434 
7435 #ifdef CONFIG_SYSFS
7436         dev->num_rx_queues = rxqs;
7437         dev->real_num_rx_queues = rxqs;
7438         if (netif_alloc_rx_queues(dev))
7439                 goto free_all;
7440 #endif
7441 
7442         strcpy(dev->name, name);
7443         dev->name_assign_type = name_assign_type;
7444         dev->group = INIT_NETDEV_GROUP;
7445         if (!dev->ethtool_ops)
7446                 dev->ethtool_ops = &default_ethtool_ops;
7447 
7448         nf_hook_ingress_init(dev);
7449 
7450         return dev;
7451 
7452 free_all:
7453         free_netdev(dev);
7454         return NULL;
7455 
7456 free_pcpu:
7457         free_percpu(dev->pcpu_refcnt);
7458 free_dev:
7459         netdev_freemem(dev);
7460         return NULL;
7461 }
7462 EXPORT_SYMBOL(alloc_netdev_mqs);
7463 
7464 /**
7465  *      free_netdev - free network device
7466  *      @dev: device
7467  *
7468  *      This function does the last stage of destroying an allocated device
7469  *      interface. The reference to the device object is released.
7470  *      If this is the last reference then it will be freed.
7471  *      Must be called in process context.
7472  */
7473 void free_netdev(struct net_device *dev)
7474 {
7475         struct napi_struct *p, *n;
7476 
7477         might_sleep();
7478         netif_free_tx_queues(dev);
7479 #ifdef CONFIG_SYSFS
7480         kvfree(dev->_rx);
7481 #endif
7482 
7483         kfree(rcu_dereference_protected(dev->ingress_queue, 1));
7484 
7485         /* Flush device addresses */
7486         dev_addr_flush(dev);
7487 
7488         list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
7489                 netif_napi_del(p);
7490 
7491         free_percpu(dev->pcpu_refcnt);
7492         dev->pcpu_refcnt = NULL;
7493 
7494         /*  Compatibility with error handling in drivers */
7495         if (dev->reg_state == NETREG_UNINITIALIZED) {
7496                 netdev_freemem(dev);
7497                 return;
7498         }
7499 
7500         BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
7501         dev->reg_state = NETREG_RELEASED;
7502 
7503         /* will free via device release */
7504         put_device(&dev->dev);
7505 }
7506 EXPORT_SYMBOL(free_netdev);
7507 
7508 /**
7509  *      synchronize_net -  Synchronize with packet receive processing
7510  *
7511  *      Wait for packets currently being received to be done.
7512  *      Does not block later packets from starting.
7513  */
7514 void synchronize_net(void)
7515 {
7516         might_sleep();
7517         if (rtnl_is_locked())
7518                 synchronize_rcu_expedited();
7519         else
7520                 synchronize_rcu();
7521 }
7522 EXPORT_SYMBOL(synchronize_net);
7523 
7524 /**
7525  *      unregister_netdevice_queue - remove device from the kernel
7526  *      @dev: device
7527  *      @head: list
7528  *
7529  *      This function shuts down a device interface and removes it
7530  *      from the kernel tables.
7531  *      If head not NULL, device is queued to be unregistered later.
7532  *
7533  *      Callers must hold the rtnl semaphore.  You may want
7534  *      unregister_netdev() instead of this.
7535  */
7536 
7537 void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
7538 {
7539         ASSERT_RTNL();
7540 
7541         if (head) {
7542                 list_move_tail(&dev->unreg_list, head);
7543         } else {
7544                 rollback_registered(dev);
7545                 /* Finish processing unregister after unlock */
7546                 net_set_todo(dev);
7547         }
7548 }
7549 EXPORT_SYMBOL(unregister_netdevice_queue);
7550 
7551 /**
7552  *      unregister_netdevice_many - unregister many devices
7553  *      @head: list of devices
7554  *
7555  *  Note: As most callers use a stack allocated list_head,
7556  *  we force a list_del() to make sure stack wont be corrupted later.
7557  */
7558 void unregister_netdevice_many(struct list_head *head)
7559 {
7560         struct net_device *dev;
7561 
7562         if (!list_empty(head)) {
7563                 rollback_registered_many(head);
7564                 list_for_each_entry(dev, head, unreg_list)
7565                         net_set_todo(dev);
7566                 list_del(head);
7567         }
7568 }
7569 EXPORT_SYMBOL(unregister_netdevice_many);
7570 
7571 /**
7572  *      unregister_netdev - remove device from the kernel
7573  *      @dev: device
7574  *
7575  *      This function shuts down a device interface and removes it
7576  *      from the kernel tables.
7577  *
7578  *      This is just a wrapper for unregister_netdevice that takes
7579  *      the rtnl semaphore.  In general you want to use this and not
7580  *      unregister_netdevice.
7581  */
7582 void unregister_netdev(struct net_device *dev)
7583 {
7584         rtnl_lock();
7585         unregister_netdevice(dev);
7586         rtnl_unlock();
7587 }
7588 EXPORT_SYMBOL(unregister_netdev);
7589 
7590 /**
7591  *      dev_change_net_namespace - move device to different nethost namespace
7592  *      @dev: device
7593  *      @net: network namespace
7594  *      @pat: If not NULL name pattern to try if the current device name
7595  *            is already taken in the destination network namespace.
7596  *
7597  *      This function shuts down a device interface and moves it
7598  *      to a new network namespace. On success 0 is returned, on
7599  *      a failure a netagive errno code is returned.
7600  *
7601  *      Callers must hold the rtnl semaphore.
7602  */
7603 
7604 int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
7605 {
7606         int err;
7607 
7608         ASSERT_RTNL();
7609 
7610         /* Don't allow namespace local devices to be moved. */
7611         err = -EINVAL;
7612         if (dev->features & NETIF_F_NETNS_LOCAL)
7613                 goto out;
7614 
7615         /* Ensure the device has been registrered */
7616         if (dev->reg_state != NETREG_REGISTERED)
7617                 goto out;
7618 
7619         /* Get out if there is nothing todo */
7620         err = 0;
7621         if (net_eq(dev_net(dev), net))
7622                 goto out;
7623 
7624         /* Pick the destination device name, and ensure
7625          * we can use it in the destination network namespace.
7626          */
7627         err = -EEXIST;
7628         if (__dev_get_by_name(net, dev->name)) {
7629                 /* We get here if we can't use the current device name */
7630                 if (!pat)
7631                         goto out;
7632                 if (dev_get_valid_name(net, dev, pat) < 0)
7633                         goto out;
7634         }
7635 
7636         /*
7637          * And now a mini version of register_netdevice unregister_netdevice.
7638          */
7639 
7640         /* If device is running close it first. */
7641         dev_close(dev);
7642 
7643         /* And unlink it from device chain */
7644         err = -ENODEV;
7645         unlist_netdevice(dev);
7646 
7647         synchronize_net();
7648 
7649         /* Shutdown queueing discipline. */
7650         dev_shutdown(dev);
7651 
7652         /* Notify protocols, that we are about to destroy
7653            this device. They should clean all the things.
7654 
7655            Note that dev->reg_state stays at NETREG_REGISTERED.
7656            This is wanted because this way 8021q and macvlan know
7657            the device is just moving and can keep their slaves up.
7658         */
7659         call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
7660         rcu_barrier();
7661         call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
7662         rtmsg_ifinfo(RTM_DELLINK, dev, ~0U, GFP_KERNEL);
7663 
7664         /*
7665          *      Flush the unicast and multicast chains
7666          */
7667         dev_uc_flush(dev);
7668         dev_mc_flush(dev);
7669 
7670         /* Send a netdev-removed uevent to the old namespace */
7671         kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);
7672         netdev_adjacent_del_links(dev);
7673 
7674         /* Actually switch the network namespace */
7675         dev_net_set(dev, net);
7676 
7677         /* If there is an ifindex conflict assign a new one */
7678         if (__dev_get_by_index(net, dev->ifindex))
7679                 dev->ifindex = dev_new_index(net);
7680 
7681         /* Send a netdev-add uevent to the new namespace */
7682         kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
7683         netdev_adjacent_add_links(dev);
7684 
7685         /* Fixup kobjects */
7686         err = device_rename(&dev->dev, dev->name);
7687         WARN_ON(err);
7688 
7689         /* Add the device back in the hashes */
7690         list_netdevice(dev);
7691 
7692         /* Notify protocols, that a new device appeared. */
7693         call_netdevice_notifiers(NETDEV_REGISTER, dev);
7694 
7695         /*
7696          *      Prevent userspace races by waiting until the network
7697          *      device is fully setup before sending notifications.
7698          */
7699         rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
7700 
7701         synchronize_net();
7702         err = 0;
7703 out:
7704         return err;
7705 }
7706 EXPORT_SYMBOL_GPL(dev_change_net_namespace);
7707 
7708 static int dev_cpu_callback(struct notifier_block *nfb,
7709                             unsigned long action,
7710                             void *ocpu)
7711 {
7712         struct sk_buff **list_skb;
7713         struct sk_buff *skb;
7714         unsigned int cpu, oldcpu = (unsigned long)ocpu;
7715         struct softnet_data *sd, *oldsd;
7716 
7717         if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
7718                 return NOTIFY_OK;
7719 
7720         local_irq_disable();
7721         cpu = smp_processor_id();
7722         sd = &per_cpu(softnet_data, cpu);
7723         oldsd = &per_cpu(softnet_data, oldcpu);
7724 
7725         /* Find end of our completion_queue. */
7726         list_skb = &sd->completion_queue;
7727         while (*list_skb)
7728                 list_skb = &(*list_skb)->next;
7729         /* Append completion queue from offline CPU. */
7730         *list_skb = oldsd->completion_queue;
7731         oldsd->completion_queue = NULL;
7732 
7733         /* Append output queue from offline CPU. */
7734         if (oldsd->output_queue) {
7735                 *sd->output_queue_tailp = oldsd->output_queue;
7736                 sd->output_queue_tailp = oldsd->output_queue_tailp;
7737                 oldsd->output_queue = NULL;
7738                 oldsd->output_queue_tailp = &oldsd->output_queue;
7739         }
7740         /* Append NAPI poll list from offline CPU, with one exception :
7741          * process_backlog() must be called by cpu owning percpu backlog.
7742          * We properly handle process_queue & input_pkt_queue later.
7743          */
7744         while (!list_empty(&oldsd->poll_list)) {
7745                 struct napi_struct *napi = list_first_entry(&oldsd->poll_list,
7746                                                             struct napi_struct,
7747                                                             poll_list);
7748 
7749                 list_del_init(&napi->poll_list);
7750                 if (napi->poll == process_backlog)
7751                         napi->state = 0;
7752                 else
7753                         ____napi_schedule(sd, napi);
7754         }
7755 
7756         raise_softirq_irqoff(NET_TX_SOFTIRQ);
7757         local_irq_enable();
7758 
7759         /* Process offline CPU's input_pkt_queue */
7760         while ((skb = __skb_dequeue(&oldsd->process_queue))) {
7761                 netif_rx_ni(skb);
7762                 input_queue_head_incr(oldsd);
7763         }
7764         while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) {
7765                 netif_rx_ni(skb);
7766                 input_queue_head_incr(oldsd);
7767         }
7768 
7769         return NOTIFY_OK;
7770 }
7771 
7772 
7773 /**
7774  *      netdev_increment_features - increment feature set by one
7775  *      @all: current feature set
7776  *      @one: new feature set
7777  *      @mask: mask feature set
7778  *
7779  *      Computes a new feature set after adding a device with feature set
7780  *      @one to the master device with current feature set @all.  Will not
7781  *      enable anything that is off in @mask. Returns the new feature set.
7782  */
7783 netdev_features_t netdev_increment_features(netdev_features_t all,
7784         netdev_features_t one, netdev_features_t mask)
7785 {
7786         if (mask & NETIF_F_HW_CSUM)
7787                 mask |= NETIF_F_CSUM_MASK;
7788         mask |= NETIF_F_VLAN_CHALLENGED;
7789 
7790         all |= one & (NETIF_F_ONE_FOR_ALL | NETIF_F_CSUM_MASK) & mask;
7791         all &= one | ~NETIF_F_ALL_FOR_ALL;
7792 
7793         /* If one device supports hw checksumming, set for all. */
7794         if (all & NETIF_F_HW_CSUM)
7795                 all &= ~(NETIF_F_CSUM_MASK & ~NETIF_F_HW_CSUM);
7796 
7797         return all;
7798 }
7799 EXPORT_SYMBOL(netdev_increment_features);
7800 
7801 static struct hlist_head * __net_init netdev_create_hash(void)
7802 {
7803         int i;
7804         struct hlist_head *hash;
7805 
7806         hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
7807         if (hash != NULL)
7808                 for (i = 0; i < NETDEV_HASHENTRIES; i++)
7809                         INIT_HLIST_HEAD(&hash[i]);
7810 
7811         return hash;
7812 }
7813 
7814 /* Initialize per network namespace state */
7815 static int __net_init netdev_init(struct net *net)
7816 {
7817         if (net != &init_net)
7818                 INIT_LIST_HEAD(&net->dev_base_head);
7819 
7820         net->dev_name_head = netdev_create_hash();
7821         if (net->dev_name_head == NULL)
7822                 goto err_name;
7823 
7824         net->dev_index_head = netdev_create_hash();
7825         if (net->dev_index_head == NULL)
7826                 goto err_idx;
7827 
7828         return 0;
7829 
7830 err_idx:
7831         kfree(net->dev_name_head);
7832 err_name:
7833         return -ENOMEM;
7834 }
7835 
7836 /**
7837  *      netdev_drivername - network driver for the device
7838  *      @dev: network device
7839  *
7840  *      Determine network driver for device.
7841  */
7842 const char *netdev_drivername(const struct net_device *dev)
7843 {
7844         const struct device_driver *driver;
7845         const struct device *parent;
7846         const char *empty = "";
7847 
7848         parent = dev->dev.parent;
7849         if (!parent)
7850                 return empty;
7851 
7852         driver = parent->driver;
7853         if (driver && driver->name)
7854                 return driver->name;
7855         return empty;
7856 }
7857 
7858 static void __netdev_printk(const char *level, const struct net_device *dev,
7859                             struct va_format *vaf)
7860 {
7861         if (dev && dev->dev.parent) {
7862                 dev_printk_emit(level[1] - '',
7863                                 dev->dev.parent,
7864                                 "%s %s %s%s: %pV",
7865                                 dev_driver_string(dev->dev.parent),
7866                                 dev_name(dev->dev.parent),
7867                                 netdev_name(dev), netdev_reg_state(dev),
7868                                 vaf);
7869         } else if (dev) {
7870                 printk("%s%s%s: %pV",
7871                        level, netdev_name(dev), netdev_reg_state(dev), vaf);
7872         } else {
7873                 printk("%s(NULL net_device): %pV", level, vaf);
7874         }
7875 }
7876 
7877 void netdev_printk(const char *level, const struct net_device *dev,
7878                    const char *format, ...)
7879 {
7880         struct va_format vaf;
7881         va_list args;
7882 
7883         va_start(args, format);
7884 
7885         vaf.fmt = format;
7886         vaf.va = &args;
7887 
7888         __netdev_printk(level, dev, &vaf);
7889 
7890         va_end(args);
7891 }
7892 EXPORT_SYMBOL(netdev_printk);
7893 
7894 #define define_netdev_printk_level(func, level)                 \
7895 void func(const struct net_device *dev, const char *fmt, ...)   \
7896 {                                                               \
7897         struct va_format vaf;                                   \
7898         va_list args;                                           \
7899                                                                 \
7900         va_start(args, fmt);                                    \
7901                                                                 \
7902         vaf.fmt = fmt;                                          \
7903         vaf.va = &args;                                         \
7904                                                                 \
7905         __netdev_printk(level, dev, &vaf);                      \
7906                                                                 \
7907         va_end(args);                                           \
7908 }                                                               \
7909 EXPORT_SYMBOL(func);
7910 
7911 define_netdev_printk_level(netdev_emerg, KERN_EMERG);
7912 define_netdev_printk_level(netdev_alert, KERN_ALERT);
7913 define_netdev_printk_level(netdev_crit, KERN_CRIT);
7914 define_netdev_printk_level(netdev_err, KERN_ERR);
7915 define_netdev_printk_level(netdev_warn, KERN_WARNING);
7916 define_netdev_printk_level(netdev_notice, KERN_NOTICE);
7917 define_netdev_printk_level(netdev_info, KERN_INFO);
7918 
7919 static void __net_exit netdev_exit(struct net *net)
7920 {
7921         kfree(net->dev_name_head);
7922         kfree(net->dev_index_head);
7923 }
7924 
7925 static struct pernet_operations __net_initdata netdev_net_ops = {
7926         .init = netdev_init,
7927         .exit = netdev_exit,
7928 };
7929 
7930 static void __net_exit default_device_exit(struct net *net)
7931 {
7932         struct net_device *dev, *aux;
7933         /*
7934          * Push all migratable network devices back to the
7935          * initial network namespace
7936          */
7937         rtnl_lock();
7938         for_each_netdev_safe(net, dev, aux) {
7939                 int err;
7940                 char fb_name[IFNAMSIZ];
7941 
7942                 /* Ignore unmoveable devices (i.e. loopback) */
7943                 if (dev->features & NETIF_F_NETNS_LOCAL)
7944                         continue;
7945 
7946                 /* Leave virtual devices for the generic cleanup */
7947                 if (dev->rtnl_link_ops)
7948                         continue;
7949 
7950                 /* Push remaining network devices to init_net */
7951                 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
7952                 err = dev_change_net_namespace(dev, &init_net, fb_name);
7953                 if (err) {
7954                         pr_emerg("%s: failed to move %s to init_net: %d\n",
7955                                  __func__, dev->name, err);
7956                         BUG();
7957                 }
7958         }
7959         rtnl_unlock();
7960 }
7961 
7962 static void __net_exit rtnl_lock_unregistering(struct list_head *net_list)
7963 {
7964         /* Return with the rtnl_lock held when there are no network
7965          * devices unregistering in any network namespace in net_list.
7966          */
7967         struct net *net;
7968         bool unregistering;
7969         DEFINE_WAIT_FUNC(wait, woken_wake_function);
7970 
7971         add_wait_queue(&netdev_unregistering_wq, &wait);
7972         for (;;) {
7973                 unregistering = false;
7974                 rtnl_lock();
7975                 list_for_each_entry(net, net_list, exit_list) {
7976                         if (net->dev_unreg_count > 0) {
7977                                 unregistering = true;
7978                                 break;
7979                         }
7980                 }
7981                 if (!unregistering)
7982                         break;
7983                 __rtnl_unlock();
7984 
7985                 wait_woken(&wait, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
7986         }
7987         remove_wait_queue(&netdev_unregistering_wq, &wait);
7988 }
7989 
7990 static void __net_exit default_device_exit_batch(struct list_head *net_list)
7991 {
7992         /* At exit all network devices most be removed from a network
7993          * namespace.  Do this in the reverse order of registration.
7994          * Do this across as many network namespaces as possible to
7995          * improve batching efficiency.
7996          */
7997         struct net_device *dev;
7998         struct net *net;
7999         LIST_HEAD(dev_kill_list);
8000 
8001         /* To prevent network device cleanup code from dereferencing
8002          * loopback devices or network devices that have been freed
8003          * wait here for all pending unregistrations to complete,
8004          * before unregistring the loopback device and allowing the
8005          * network namespace be freed.
8006          *
8007          * The netdev todo list containing all network devices
8008          * unregistrations that happen in default_device_exit_batch
8009          * will run in the rtnl_unlock() at the end of
8010          * default_device_exit_batch.
8011          */
8012         rtnl_lock_unregistering(net_list);
8013         list_for_each_entry(net, net_list, exit_list) {
8014                 for_each_netdev_reverse(net, dev) {
8015                         if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink)
8016                                 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
8017                         else
8018                                 unregister_netdevice_queue(dev, &dev_kill_list);
8019                 }
8020         }
8021         unregister_netdevice_many(&dev_kill_list);
8022         rtnl_unlock();
8023 }
8024 
8025 static struct pernet_operations __net_initdata default_device_ops = {
8026         .exit = default_device_exit,
8027         .exit_batch = default_device_exit_batch,
8028 };
8029 
8030 /*
8031  *      Initialize the DEV module. At boot time this walks the device list and
8032  *      unhooks any devices that fail to initialise (normally hardware not
8033  *      present) and leaves us with a valid list of present and active devices.
8034  *
8035  */
8036 
8037 /*
8038  *       This is called single threaded during boot, so no need
8039  *       to take the rtnl semaphore.
8040  */
8041 static int __init net_dev_init(void)
8042 {
8043         int i, rc = -ENOMEM;
8044 
8045         BUG_ON(!dev_boot_phase);
8046 
8047         if (dev_proc_init())
8048                 goto out;
8049 
8050         if (netdev_kobject_init())
8051                 goto out;
8052 
8053         INIT_LIST_HEAD(&ptype_all);
8054         for (i = 0; i < PTYPE_HASH_SIZE; i++)
8055                 INIT_LIST_HEAD(&ptype_base[i]);
8056 
8057         INIT_LIST_HEAD(&offload_base);
8058 
8059         if (register_pernet_subsys(&netdev_net_ops))
8060                 goto out;
8061 
8062         /*
8063          *      Initialise the packet receive queues.
8064          */
8065 
8066         for_each_possible_cpu(i) {
8067                 struct softnet_data *sd = &per_cpu(softnet_data, i);
8068 
8069                 skb_queue_head_init(&sd->input_pkt_queue);
8070                 skb_queue_head_init(&sd->process_queue);
8071                 INIT_LIST_HEAD(&sd->poll_list);
8072                 sd->output_queue_tailp = &sd->output_queue;
8073 #ifdef CONFIG_RPS
8074                 sd->csd.func = rps_trigger_softirq;
8075                 sd->csd.info = sd;
8076                 sd->cpu = i;
8077 #endif
8078 
8079                 sd->backlog.poll = process_backlog;
8080                 sd->backlog.weight = weight_p;
8081         }
8082 
8083         dev_boot_phase = 0;
8084 
8085         /* The loopback device is special if any other network devices
8086          * is present in a network namespace the loopback device must
8087          * be present. Since we now dynamically allocate and free the
8088          * loopback device ensure this invariant is maintained by
8089          * keeping the loopback device as the first device on the
8090          * list of network devices.  Ensuring the loopback devices
8091          * is the first device that appears and the last network device
8092          * that disappears.
8093          */
8094         if (register_pernet_device(&loopback_net_ops))
8095                 goto out;
8096 
8097         if (register_pernet_device(&default_device_ops))
8098                 goto out;
8099 
8100         open_softirq(NET_TX_SOFTIRQ, net_tx_action);
8101         open_softirq(NET_RX_SOFTIRQ, net_rx_action);
8102 
8103         hotcpu_notifier(dev_cpu_callback, 0);
8104         dst_subsys_init();
8105         rc = 0;
8106 out:
8107         return rc;
8108 }
8109 
8110 subsys_initcall(net_dev_init);
8111 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us