Version:  2.0.40 2.2.26 2.4.37 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1 4.2

Linux/net/core/dev.c

  1 /*
  2  *      NET3    Protocol independent device support routines.
  3  *
  4  *              This program is free software; you can redistribute it and/or
  5  *              modify it under the terms of the GNU General Public License
  6  *              as published by the Free Software Foundation; either version
  7  *              2 of the License, or (at your option) any later version.
  8  *
  9  *      Derived from the non IP parts of dev.c 1.0.19
 10  *              Authors:        Ross Biro
 11  *                              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
 12  *                              Mark Evans, <evansmp@uhura.aston.ac.uk>
 13  *
 14  *      Additional Authors:
 15  *              Florian la Roche <rzsfl@rz.uni-sb.de>
 16  *              Alan Cox <gw4pts@gw4pts.ampr.org>
 17  *              David Hinds <dahinds@users.sourceforge.net>
 18  *              Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
 19  *              Adam Sulmicki <adam@cfar.umd.edu>
 20  *              Pekka Riikonen <priikone@poesidon.pspt.fi>
 21  *
 22  *      Changes:
 23  *              D.J. Barrow     :       Fixed bug where dev->refcnt gets set
 24  *                                      to 2 if register_netdev gets called
 25  *                                      before net_dev_init & also removed a
 26  *                                      few lines of code in the process.
 27  *              Alan Cox        :       device private ioctl copies fields back.
 28  *              Alan Cox        :       Transmit queue code does relevant
 29  *                                      stunts to keep the queue safe.
 30  *              Alan Cox        :       Fixed double lock.
 31  *              Alan Cox        :       Fixed promisc NULL pointer trap
 32  *              ????????        :       Support the full private ioctl range
 33  *              Alan Cox        :       Moved ioctl permission check into
 34  *                                      drivers
 35  *              Tim Kordas      :       SIOCADDMULTI/SIOCDELMULTI
 36  *              Alan Cox        :       100 backlog just doesn't cut it when
 37  *                                      you start doing multicast video 8)
 38  *              Alan Cox        :       Rewrote net_bh and list manager.
 39  *              Alan Cox        :       Fix ETH_P_ALL echoback lengths.
 40  *              Alan Cox        :       Took out transmit every packet pass
 41  *                                      Saved a few bytes in the ioctl handler
 42  *              Alan Cox        :       Network driver sets packet type before
 43  *                                      calling netif_rx. Saves a function
 44  *                                      call a packet.
 45  *              Alan Cox        :       Hashed net_bh()
 46  *              Richard Kooijman:       Timestamp fixes.
 47  *              Alan Cox        :       Wrong field in SIOCGIFDSTADDR
 48  *              Alan Cox        :       Device lock protection.
 49  *              Alan Cox        :       Fixed nasty side effect of device close
 50  *                                      changes.
 51  *              Rudi Cilibrasi  :       Pass the right thing to
 52  *                                      set_mac_address()
 53  *              Dave Miller     :       32bit quantity for the device lock to
 54  *                                      make it work out on a Sparc.
 55  *              Bjorn Ekwall    :       Added KERNELD hack.
 56  *              Alan Cox        :       Cleaned up the backlog initialise.
 57  *              Craig Metz      :       SIOCGIFCONF fix if space for under
 58  *                                      1 device.
 59  *          Thomas Bogendoerfer :       Return ENODEV for dev_open, if there
 60  *                                      is no device open function.
 61  *              Andi Kleen      :       Fix error reporting for SIOCGIFCONF
 62  *          Michael Chastain    :       Fix signed/unsigned for SIOCGIFCONF
 63  *              Cyrus Durgin    :       Cleaned for KMOD
 64  *              Adam Sulmicki   :       Bug Fix : Network Device Unload
 65  *                                      A network device unload needs to purge
 66  *                                      the backlog queue.
 67  *      Paul Rusty Russell      :       SIOCSIFNAME
 68  *              Pekka Riikonen  :       Netdev boot-time settings code
 69  *              Andrew Morton   :       Make unregister_netdevice wait
 70  *                                      indefinitely on dev->refcnt
 71  *              J Hadi Salim    :       - Backlog queue sampling
 72  *                                      - netif_rx() feedback
 73  */
 74 
 75 #include <asm/uaccess.h>
 76 #include <linux/bitops.h>
 77 #include <linux/capability.h>
 78 #include <linux/cpu.h>
 79 #include <linux/types.h>
 80 #include <linux/kernel.h>
 81 #include <linux/hash.h>
 82 #include <linux/slab.h>
 83 #include <linux/sched.h>
 84 #include <linux/mutex.h>
 85 #include <linux/string.h>
 86 #include <linux/mm.h>
 87 #include <linux/socket.h>
 88 #include <linux/sockios.h>
 89 #include <linux/errno.h>
 90 #include <linux/interrupt.h>
 91 #include <linux/if_ether.h>
 92 #include <linux/netdevice.h>
 93 #include <linux/etherdevice.h>
 94 #include <linux/ethtool.h>
 95 #include <linux/notifier.h>
 96 #include <linux/skbuff.h>
 97 #include <net/net_namespace.h>
 98 #include <net/sock.h>
 99 #include <linux/rtnetlink.h>
100 #include <linux/stat.h>
101 #include <net/dst.h>
102 #include <net/pkt_sched.h>
103 #include <net/checksum.h>
104 #include <net/xfrm.h>
105 #include <linux/highmem.h>
106 #include <linux/init.h>
107 #include <linux/module.h>
108 #include <linux/netpoll.h>
109 #include <linux/rcupdate.h>
110 #include <linux/delay.h>
111 #include <net/iw_handler.h>
112 #include <asm/current.h>
113 #include <linux/audit.h>
114 #include <linux/dmaengine.h>
115 #include <linux/err.h>
116 #include <linux/ctype.h>
117 #include <linux/if_arp.h>
118 #include <linux/if_vlan.h>
119 #include <linux/ip.h>
120 #include <net/ip.h>
121 #include <net/mpls.h>
122 #include <linux/ipv6.h>
123 #include <linux/in.h>
124 #include <linux/jhash.h>
125 #include <linux/random.h>
126 #include <trace/events/napi.h>
127 #include <trace/events/net.h>
128 #include <trace/events/skb.h>
129 #include <linux/pci.h>
130 #include <linux/inetdevice.h>
131 #include <linux/cpu_rmap.h>
132 #include <linux/static_key.h>
133 #include <linux/hashtable.h>
134 #include <linux/vmalloc.h>
135 #include <linux/if_macvlan.h>
136 #include <linux/errqueue.h>
137 #include <linux/hrtimer.h>
138 #include <linux/netfilter_ingress.h>
139 
140 #include "net-sysfs.h"
141 
142 /* Instead of increasing this, you should create a hash table. */
143 #define MAX_GRO_SKBS 8
144 
145 /* This should be increased if a protocol with a bigger head is added. */
146 #define GRO_MAX_HEAD (MAX_HEADER + 128)
147 
148 static DEFINE_SPINLOCK(ptype_lock);
149 static DEFINE_SPINLOCK(offload_lock);
150 struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
151 struct list_head ptype_all __read_mostly;       /* Taps */
152 static struct list_head offload_base __read_mostly;
153 
154 static int netif_rx_internal(struct sk_buff *skb);
155 static int call_netdevice_notifiers_info(unsigned long val,
156                                          struct net_device *dev,
157                                          struct netdev_notifier_info *info);
158 
159 /*
160  * The @dev_base_head list is protected by @dev_base_lock and the rtnl
161  * semaphore.
162  *
163  * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
164  *
165  * Writers must hold the rtnl semaphore while they loop through the
166  * dev_base_head list, and hold dev_base_lock for writing when they do the
167  * actual updates.  This allows pure readers to access the list even
168  * while a writer is preparing to update it.
169  *
170  * To put it another way, dev_base_lock is held for writing only to
171  * protect against pure readers; the rtnl semaphore provides the
172  * protection against other writers.
173  *
174  * See, for example usages, register_netdevice() and
175  * unregister_netdevice(), which must be called with the rtnl
176  * semaphore held.
177  */
178 DEFINE_RWLOCK(dev_base_lock);
179 EXPORT_SYMBOL(dev_base_lock);
180 
181 /* protects napi_hash addition/deletion and napi_gen_id */
182 static DEFINE_SPINLOCK(napi_hash_lock);
183 
184 static unsigned int napi_gen_id;
185 static DEFINE_HASHTABLE(napi_hash, 8);
186 
187 static seqcount_t devnet_rename_seq;
188 
189 static inline void dev_base_seq_inc(struct net *net)
190 {
191         while (++net->dev_base_seq == 0);
192 }
193 
194 static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
195 {
196         unsigned int hash = full_name_hash(name, strnlen(name, IFNAMSIZ));
197 
198         return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
199 }
200 
201 static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
202 {
203         return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
204 }
205 
206 static inline void rps_lock(struct softnet_data *sd)
207 {
208 #ifdef CONFIG_RPS
209         spin_lock(&sd->input_pkt_queue.lock);
210 #endif
211 }
212 
213 static inline void rps_unlock(struct softnet_data *sd)
214 {
215 #ifdef CONFIG_RPS
216         spin_unlock(&sd->input_pkt_queue.lock);
217 #endif
218 }
219 
220 /* Device list insertion */
221 static void list_netdevice(struct net_device *dev)
222 {
223         struct net *net = dev_net(dev);
224 
225         ASSERT_RTNL();
226 
227         write_lock_bh(&dev_base_lock);
228         list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
229         hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
230         hlist_add_head_rcu(&dev->index_hlist,
231                            dev_index_hash(net, dev->ifindex));
232         write_unlock_bh(&dev_base_lock);
233 
234         dev_base_seq_inc(net);
235 }
236 
237 /* Device list removal
238  * caller must respect a RCU grace period before freeing/reusing dev
239  */
240 static void unlist_netdevice(struct net_device *dev)
241 {
242         ASSERT_RTNL();
243 
244         /* Unlink dev from the device chain */
245         write_lock_bh(&dev_base_lock);
246         list_del_rcu(&dev->dev_list);
247         hlist_del_rcu(&dev->name_hlist);
248         hlist_del_rcu(&dev->index_hlist);
249         write_unlock_bh(&dev_base_lock);
250 
251         dev_base_seq_inc(dev_net(dev));
252 }
253 
254 /*
255  *      Our notifier list
256  */
257 
258 static RAW_NOTIFIER_HEAD(netdev_chain);
259 
260 /*
261  *      Device drivers call our routines to queue packets here. We empty the
262  *      queue in the local softnet handler.
263  */
264 
265 DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
266 EXPORT_PER_CPU_SYMBOL(softnet_data);
267 
268 #ifdef CONFIG_LOCKDEP
269 /*
270  * register_netdevice() inits txq->_xmit_lock and sets lockdep class
271  * according to dev->type
272  */
273 static const unsigned short netdev_lock_type[] =
274         {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
275          ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
276          ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
277          ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
278          ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
279          ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
280          ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
281          ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
282          ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
283          ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
284          ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
285          ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
286          ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
287          ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
288          ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
289 
290 static const char *const netdev_lock_name[] =
291         {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
292          "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
293          "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
294          "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
295          "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
296          "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
297          "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
298          "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
299          "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
300          "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
301          "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
302          "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
303          "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
304          "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
305          "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
306 
307 static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
308 static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
309 
310 static inline unsigned short netdev_lock_pos(unsigned short dev_type)
311 {
312         int i;
313 
314         for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
315                 if (netdev_lock_type[i] == dev_type)
316                         return i;
317         /* the last key is used by default */
318         return ARRAY_SIZE(netdev_lock_type) - 1;
319 }
320 
321 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
322                                                  unsigned short dev_type)
323 {
324         int i;
325 
326         i = netdev_lock_pos(dev_type);
327         lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
328                                    netdev_lock_name[i]);
329 }
330 
331 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
332 {
333         int i;
334 
335         i = netdev_lock_pos(dev->type);
336         lockdep_set_class_and_name(&dev->addr_list_lock,
337                                    &netdev_addr_lock_key[i],
338                                    netdev_lock_name[i]);
339 }
340 #else
341 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
342                                                  unsigned short dev_type)
343 {
344 }
345 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
346 {
347 }
348 #endif
349 
350 /*******************************************************************************
351 
352                 Protocol management and registration routines
353 
354 *******************************************************************************/
355 
356 /*
357  *      Add a protocol ID to the list. Now that the input handler is
358  *      smarter we can dispense with all the messy stuff that used to be
359  *      here.
360  *
361  *      BEWARE!!! Protocol handlers, mangling input packets,
362  *      MUST BE last in hash buckets and checking protocol handlers
363  *      MUST start from promiscuous ptype_all chain in net_bh.
364  *      It is true now, do not change it.
365  *      Explanation follows: if protocol handler, mangling packet, will
366  *      be the first on list, it is not able to sense, that packet
367  *      is cloned and should be copied-on-write, so that it will
368  *      change it and subsequent readers will get broken packet.
369  *                                                      --ANK (980803)
370  */
371 
372 static inline struct list_head *ptype_head(const struct packet_type *pt)
373 {
374         if (pt->type == htons(ETH_P_ALL))
375                 return pt->dev ? &pt->dev->ptype_all : &ptype_all;
376         else
377                 return pt->dev ? &pt->dev->ptype_specific :
378                                  &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
379 }
380 
381 /**
382  *      dev_add_pack - add packet handler
383  *      @pt: packet type declaration
384  *
385  *      Add a protocol handler to the networking stack. The passed &packet_type
386  *      is linked into kernel lists and may not be freed until it has been
387  *      removed from the kernel lists.
388  *
389  *      This call does not sleep therefore it can not
390  *      guarantee all CPU's that are in middle of receiving packets
391  *      will see the new packet type (until the next received packet).
392  */
393 
394 void dev_add_pack(struct packet_type *pt)
395 {
396         struct list_head *head = ptype_head(pt);
397 
398         spin_lock(&ptype_lock);
399         list_add_rcu(&pt->list, head);
400         spin_unlock(&ptype_lock);
401 }
402 EXPORT_SYMBOL(dev_add_pack);
403 
404 /**
405  *      __dev_remove_pack        - remove packet handler
406  *      @pt: packet type declaration
407  *
408  *      Remove a protocol handler that was previously added to the kernel
409  *      protocol handlers by dev_add_pack(). The passed &packet_type is removed
410  *      from the kernel lists and can be freed or reused once this function
411  *      returns.
412  *
413  *      The packet type might still be in use by receivers
414  *      and must not be freed until after all the CPU's have gone
415  *      through a quiescent state.
416  */
417 void __dev_remove_pack(struct packet_type *pt)
418 {
419         struct list_head *head = ptype_head(pt);
420         struct packet_type *pt1;
421 
422         spin_lock(&ptype_lock);
423 
424         list_for_each_entry(pt1, head, list) {
425                 if (pt == pt1) {
426                         list_del_rcu(&pt->list);
427                         goto out;
428                 }
429         }
430 
431         pr_warn("dev_remove_pack: %p not found\n", pt);
432 out:
433         spin_unlock(&ptype_lock);
434 }
435 EXPORT_SYMBOL(__dev_remove_pack);
436 
437 /**
438  *      dev_remove_pack  - remove packet handler
439  *      @pt: packet type declaration
440  *
441  *      Remove a protocol handler that was previously added to the kernel
442  *      protocol handlers by dev_add_pack(). The passed &packet_type is removed
443  *      from the kernel lists and can be freed or reused once this function
444  *      returns.
445  *
446  *      This call sleeps to guarantee that no CPU is looking at the packet
447  *      type after return.
448  */
449 void dev_remove_pack(struct packet_type *pt)
450 {
451         __dev_remove_pack(pt);
452 
453         synchronize_net();
454 }
455 EXPORT_SYMBOL(dev_remove_pack);
456 
457 
458 /**
459  *      dev_add_offload - register offload handlers
460  *      @po: protocol offload declaration
461  *
462  *      Add protocol offload handlers to the networking stack. The passed
463  *      &proto_offload is linked into kernel lists and may not be freed until
464  *      it has been removed from the kernel lists.
465  *
466  *      This call does not sleep therefore it can not
467  *      guarantee all CPU's that are in middle of receiving packets
468  *      will see the new offload handlers (until the next received packet).
469  */
470 void dev_add_offload(struct packet_offload *po)
471 {
472         struct packet_offload *elem;
473 
474         spin_lock(&offload_lock);
475         list_for_each_entry(elem, &offload_base, list) {
476                 if (po->priority < elem->priority)
477                         break;
478         }
479         list_add_rcu(&po->list, elem->list.prev);
480         spin_unlock(&offload_lock);
481 }
482 EXPORT_SYMBOL(dev_add_offload);
483 
484 /**
485  *      __dev_remove_offload     - remove offload handler
486  *      @po: packet offload declaration
487  *
488  *      Remove a protocol offload handler that was previously added to the
489  *      kernel offload handlers by dev_add_offload(). The passed &offload_type
490  *      is removed from the kernel lists and can be freed or reused once this
491  *      function returns.
492  *
493  *      The packet type might still be in use by receivers
494  *      and must not be freed until after all the CPU's have gone
495  *      through a quiescent state.
496  */
497 static void __dev_remove_offload(struct packet_offload *po)
498 {
499         struct list_head *head = &offload_base;
500         struct packet_offload *po1;
501 
502         spin_lock(&offload_lock);
503 
504         list_for_each_entry(po1, head, list) {
505                 if (po == po1) {
506                         list_del_rcu(&po->list);
507                         goto out;
508                 }
509         }
510 
511         pr_warn("dev_remove_offload: %p not found\n", po);
512 out:
513         spin_unlock(&offload_lock);
514 }
515 
516 /**
517  *      dev_remove_offload       - remove packet offload handler
518  *      @po: packet offload declaration
519  *
520  *      Remove a packet offload handler that was previously added to the kernel
521  *      offload handlers by dev_add_offload(). The passed &offload_type is
522  *      removed from the kernel lists and can be freed or reused once this
523  *      function returns.
524  *
525  *      This call sleeps to guarantee that no CPU is looking at the packet
526  *      type after return.
527  */
528 void dev_remove_offload(struct packet_offload *po)
529 {
530         __dev_remove_offload(po);
531 
532         synchronize_net();
533 }
534 EXPORT_SYMBOL(dev_remove_offload);
535 
536 /******************************************************************************
537 
538                       Device Boot-time Settings Routines
539 
540 *******************************************************************************/
541 
542 /* Boot time configuration table */
543 static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
544 
545 /**
546  *      netdev_boot_setup_add   - add new setup entry
547  *      @name: name of the device
548  *      @map: configured settings for the device
549  *
550  *      Adds new setup entry to the dev_boot_setup list.  The function
551  *      returns 0 on error and 1 on success.  This is a generic routine to
552  *      all netdevices.
553  */
554 static int netdev_boot_setup_add(char *name, struct ifmap *map)
555 {
556         struct netdev_boot_setup *s;
557         int i;
558 
559         s = dev_boot_setup;
560         for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
561                 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
562                         memset(s[i].name, 0, sizeof(s[i].name));
563                         strlcpy(s[i].name, name, IFNAMSIZ);
564                         memcpy(&s[i].map, map, sizeof(s[i].map));
565                         break;
566                 }
567         }
568 
569         return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
570 }
571 
572 /**
573  *      netdev_boot_setup_check - check boot time settings
574  *      @dev: the netdevice
575  *
576  *      Check boot time settings for the device.
577  *      The found settings are set for the device to be used
578  *      later in the device probing.
579  *      Returns 0 if no settings found, 1 if they are.
580  */
581 int netdev_boot_setup_check(struct net_device *dev)
582 {
583         struct netdev_boot_setup *s = dev_boot_setup;
584         int i;
585 
586         for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
587                 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
588                     !strcmp(dev->name, s[i].name)) {
589                         dev->irq        = s[i].map.irq;
590                         dev->base_addr  = s[i].map.base_addr;
591                         dev->mem_start  = s[i].map.mem_start;
592                         dev->mem_end    = s[i].map.mem_end;
593                         return 1;
594                 }
595         }
596         return 0;
597 }
598 EXPORT_SYMBOL(netdev_boot_setup_check);
599 
600 
601 /**
602  *      netdev_boot_base        - get address from boot time settings
603  *      @prefix: prefix for network device
604  *      @unit: id for network device
605  *
606  *      Check boot time settings for the base address of device.
607  *      The found settings are set for the device to be used
608  *      later in the device probing.
609  *      Returns 0 if no settings found.
610  */
611 unsigned long netdev_boot_base(const char *prefix, int unit)
612 {
613         const struct netdev_boot_setup *s = dev_boot_setup;
614         char name[IFNAMSIZ];
615         int i;
616 
617         sprintf(name, "%s%d", prefix, unit);
618 
619         /*
620          * If device already registered then return base of 1
621          * to indicate not to probe for this interface
622          */
623         if (__dev_get_by_name(&init_net, name))
624                 return 1;
625 
626         for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
627                 if (!strcmp(name, s[i].name))
628                         return s[i].map.base_addr;
629         return 0;
630 }
631 
632 /*
633  * Saves at boot time configured settings for any netdevice.
634  */
635 int __init netdev_boot_setup(char *str)
636 {
637         int ints[5];
638         struct ifmap map;
639 
640         str = get_options(str, ARRAY_SIZE(ints), ints);
641         if (!str || !*str)
642                 return 0;
643 
644         /* Save settings */
645         memset(&map, 0, sizeof(map));
646         if (ints[0] > 0)
647                 map.irq = ints[1];
648         if (ints[0] > 1)
649                 map.base_addr = ints[2];
650         if (ints[0] > 2)
651                 map.mem_start = ints[3];
652         if (ints[0] > 3)
653                 map.mem_end = ints[4];
654 
655         /* Add new entry to the list */
656         return netdev_boot_setup_add(str, &map);
657 }
658 
659 __setup("netdev=", netdev_boot_setup);
660 
661 /*******************************************************************************
662 
663                             Device Interface Subroutines
664 
665 *******************************************************************************/
666 
667 /**
668  *      dev_get_iflink  - get 'iflink' value of a interface
669  *      @dev: targeted interface
670  *
671  *      Indicates the ifindex the interface is linked to.
672  *      Physical interfaces have the same 'ifindex' and 'iflink' values.
673  */
674 
675 int dev_get_iflink(const struct net_device *dev)
676 {
677         if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
678                 return dev->netdev_ops->ndo_get_iflink(dev);
679 
680         return dev->ifindex;
681 }
682 EXPORT_SYMBOL(dev_get_iflink);
683 
684 /**
685  *      __dev_get_by_name       - find a device by its name
686  *      @net: the applicable net namespace
687  *      @name: name to find
688  *
689  *      Find an interface by name. Must be called under RTNL semaphore
690  *      or @dev_base_lock. If the name is found a pointer to the device
691  *      is returned. If the name is not found then %NULL is returned. The
692  *      reference counters are not incremented so the caller must be
693  *      careful with locks.
694  */
695 
696 struct net_device *__dev_get_by_name(struct net *net, const char *name)
697 {
698         struct net_device *dev;
699         struct hlist_head *head = dev_name_hash(net, name);
700 
701         hlist_for_each_entry(dev, head, name_hlist)
702                 if (!strncmp(dev->name, name, IFNAMSIZ))
703                         return dev;
704 
705         return NULL;
706 }
707 EXPORT_SYMBOL(__dev_get_by_name);
708 
709 /**
710  *      dev_get_by_name_rcu     - find a device by its name
711  *      @net: the applicable net namespace
712  *      @name: name to find
713  *
714  *      Find an interface by name.
715  *      If the name is found a pointer to the device is returned.
716  *      If the name is not found then %NULL is returned.
717  *      The reference counters are not incremented so the caller must be
718  *      careful with locks. The caller must hold RCU lock.
719  */
720 
721 struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
722 {
723         struct net_device *dev;
724         struct hlist_head *head = dev_name_hash(net, name);
725 
726         hlist_for_each_entry_rcu(dev, head, name_hlist)
727                 if (!strncmp(dev->name, name, IFNAMSIZ))
728                         return dev;
729 
730         return NULL;
731 }
732 EXPORT_SYMBOL(dev_get_by_name_rcu);
733 
734 /**
735  *      dev_get_by_name         - find a device by its name
736  *      @net: the applicable net namespace
737  *      @name: name to find
738  *
739  *      Find an interface by name. This can be called from any
740  *      context and does its own locking. The returned handle has
741  *      the usage count incremented and the caller must use dev_put() to
742  *      release it when it is no longer needed. %NULL is returned if no
743  *      matching device is found.
744  */
745 
746 struct net_device *dev_get_by_name(struct net *net, const char *name)
747 {
748         struct net_device *dev;
749 
750         rcu_read_lock();
751         dev = dev_get_by_name_rcu(net, name);
752         if (dev)
753                 dev_hold(dev);
754         rcu_read_unlock();
755         return dev;
756 }
757 EXPORT_SYMBOL(dev_get_by_name);
758 
759 /**
760  *      __dev_get_by_index - find a device by its ifindex
761  *      @net: the applicable net namespace
762  *      @ifindex: index of device
763  *
764  *      Search for an interface by index. Returns %NULL if the device
765  *      is not found or a pointer to the device. The device has not
766  *      had its reference counter increased so the caller must be careful
767  *      about locking. The caller must hold either the RTNL semaphore
768  *      or @dev_base_lock.
769  */
770 
771 struct net_device *__dev_get_by_index(struct net *net, int ifindex)
772 {
773         struct net_device *dev;
774         struct hlist_head *head = dev_index_hash(net, ifindex);
775 
776         hlist_for_each_entry(dev, head, index_hlist)
777                 if (dev->ifindex == ifindex)
778                         return dev;
779 
780         return NULL;
781 }
782 EXPORT_SYMBOL(__dev_get_by_index);
783 
784 /**
785  *      dev_get_by_index_rcu - find a device by its ifindex
786  *      @net: the applicable net namespace
787  *      @ifindex: index of device
788  *
789  *      Search for an interface by index. Returns %NULL if the device
790  *      is not found or a pointer to the device. The device has not
791  *      had its reference counter increased so the caller must be careful
792  *      about locking. The caller must hold RCU lock.
793  */
794 
795 struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
796 {
797         struct net_device *dev;
798         struct hlist_head *head = dev_index_hash(net, ifindex);
799 
800         hlist_for_each_entry_rcu(dev, head, index_hlist)
801                 if (dev->ifindex == ifindex)
802                         return dev;
803 
804         return NULL;
805 }
806 EXPORT_SYMBOL(dev_get_by_index_rcu);
807 
808 
809 /**
810  *      dev_get_by_index - find a device by its ifindex
811  *      @net: the applicable net namespace
812  *      @ifindex: index of device
813  *
814  *      Search for an interface by index. Returns NULL if the device
815  *      is not found or a pointer to the device. The device returned has
816  *      had a reference added and the pointer is safe until the user calls
817  *      dev_put to indicate they have finished with it.
818  */
819 
820 struct net_device *dev_get_by_index(struct net *net, int ifindex)
821 {
822         struct net_device *dev;
823 
824         rcu_read_lock();
825         dev = dev_get_by_index_rcu(net, ifindex);
826         if (dev)
827                 dev_hold(dev);
828         rcu_read_unlock();
829         return dev;
830 }
831 EXPORT_SYMBOL(dev_get_by_index);
832 
833 /**
834  *      netdev_get_name - get a netdevice name, knowing its ifindex.
835  *      @net: network namespace
836  *      @name: a pointer to the buffer where the name will be stored.
837  *      @ifindex: the ifindex of the interface to get the name from.
838  *
839  *      The use of raw_seqcount_begin() and cond_resched() before
840  *      retrying is required as we want to give the writers a chance
841  *      to complete when CONFIG_PREEMPT is not set.
842  */
843 int netdev_get_name(struct net *net, char *name, int ifindex)
844 {
845         struct net_device *dev;
846         unsigned int seq;
847 
848 retry:
849         seq = raw_seqcount_begin(&devnet_rename_seq);
850         rcu_read_lock();
851         dev = dev_get_by_index_rcu(net, ifindex);
852         if (!dev) {
853                 rcu_read_unlock();
854                 return -ENODEV;
855         }
856 
857         strcpy(name, dev->name);
858         rcu_read_unlock();
859         if (read_seqcount_retry(&devnet_rename_seq, seq)) {
860                 cond_resched();
861                 goto retry;
862         }
863 
864         return 0;
865 }
866 
867 /**
868  *      dev_getbyhwaddr_rcu - find a device by its hardware address
869  *      @net: the applicable net namespace
870  *      @type: media type of device
871  *      @ha: hardware address
872  *
873  *      Search for an interface by MAC address. Returns NULL if the device
874  *      is not found or a pointer to the device.
875  *      The caller must hold RCU or RTNL.
876  *      The returned device has not had its ref count increased
877  *      and the caller must therefore be careful about locking
878  *
879  */
880 
881 struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
882                                        const char *ha)
883 {
884         struct net_device *dev;
885 
886         for_each_netdev_rcu(net, dev)
887                 if (dev->type == type &&
888                     !memcmp(dev->dev_addr, ha, dev->addr_len))
889                         return dev;
890 
891         return NULL;
892 }
893 EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
894 
895 struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
896 {
897         struct net_device *dev;
898 
899         ASSERT_RTNL();
900         for_each_netdev(net, dev)
901                 if (dev->type == type)
902                         return dev;
903 
904         return NULL;
905 }
906 EXPORT_SYMBOL(__dev_getfirstbyhwtype);
907 
908 struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
909 {
910         struct net_device *dev, *ret = NULL;
911 
912         rcu_read_lock();
913         for_each_netdev_rcu(net, dev)
914                 if (dev->type == type) {
915                         dev_hold(dev);
916                         ret = dev;
917                         break;
918                 }
919         rcu_read_unlock();
920         return ret;
921 }
922 EXPORT_SYMBOL(dev_getfirstbyhwtype);
923 
924 /**
925  *      __dev_get_by_flags - find any device with given flags
926  *      @net: the applicable net namespace
927  *      @if_flags: IFF_* values
928  *      @mask: bitmask of bits in if_flags to check
929  *
930  *      Search for any interface with the given flags. Returns NULL if a device
931  *      is not found or a pointer to the device. Must be called inside
932  *      rtnl_lock(), and result refcount is unchanged.
933  */
934 
935 struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
936                                       unsigned short mask)
937 {
938         struct net_device *dev, *ret;
939 
940         ASSERT_RTNL();
941 
942         ret = NULL;
943         for_each_netdev(net, dev) {
944                 if (((dev->flags ^ if_flags) & mask) == 0) {
945                         ret = dev;
946                         break;
947                 }
948         }
949         return ret;
950 }
951 EXPORT_SYMBOL(__dev_get_by_flags);
952 
953 /**
954  *      dev_valid_name - check if name is okay for network device
955  *      @name: name string
956  *
957  *      Network device names need to be valid file names to
958  *      to allow sysfs to work.  We also disallow any kind of
959  *      whitespace.
960  */
961 bool dev_valid_name(const char *name)
962 {
963         if (*name == '\0')
964                 return false;
965         if (strlen(name) >= IFNAMSIZ)
966                 return false;
967         if (!strcmp(name, ".") || !strcmp(name, ".."))
968                 return false;
969 
970         while (*name) {
971                 if (*name == '/' || *name == ':' || isspace(*name))
972                         return false;
973                 name++;
974         }
975         return true;
976 }
977 EXPORT_SYMBOL(dev_valid_name);
978 
979 /**
980  *      __dev_alloc_name - allocate a name for a device
981  *      @net: network namespace to allocate the device name in
982  *      @name: name format string
983  *      @buf:  scratch buffer and result name string
984  *
985  *      Passed a format string - eg "lt%d" it will try and find a suitable
986  *      id. It scans list of devices to build up a free map, then chooses
987  *      the first empty slot. The caller must hold the dev_base or rtnl lock
988  *      while allocating the name and adding the device in order to avoid
989  *      duplicates.
990  *      Limited to bits_per_byte * page size devices (ie 32K on most platforms).
991  *      Returns the number of the unit assigned or a negative errno code.
992  */
993 
994 static int __dev_alloc_name(struct net *net, const char *name, char *buf)
995 {
996         int i = 0;
997         const char *p;
998         const int max_netdevices = 8*PAGE_SIZE;
999         unsigned long *inuse;
1000         struct net_device *d;
1001 
1002         p = strnchr(name, IFNAMSIZ-1, '%');
1003         if (p) {
1004                 /*
1005                  * Verify the string as this thing may have come from
1006                  * the user.  There must be either one "%d" and no other "%"
1007                  * characters.
1008                  */
1009                 if (p[1] != 'd' || strchr(p + 2, '%'))
1010                         return -EINVAL;
1011 
1012                 /* Use one page as a bit array of possible slots */
1013                 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
1014                 if (!inuse)
1015                         return -ENOMEM;
1016 
1017                 for_each_netdev(net, d) {
1018                         if (!sscanf(d->name, name, &i))
1019                                 continue;
1020                         if (i < 0 || i >= max_netdevices)
1021                                 continue;
1022 
1023                         /*  avoid cases where sscanf is not exact inverse of printf */
1024                         snprintf(buf, IFNAMSIZ, name, i);
1025                         if (!strncmp(buf, d->name, IFNAMSIZ))
1026                                 set_bit(i, inuse);
1027                 }
1028 
1029                 i = find_first_zero_bit(inuse, max_netdevices);
1030                 free_page((unsigned long) inuse);
1031         }
1032 
1033         if (buf != name)
1034                 snprintf(buf, IFNAMSIZ, name, i);
1035         if (!__dev_get_by_name(net, buf))
1036                 return i;
1037 
1038         /* It is possible to run out of possible slots
1039          * when the name is long and there isn't enough space left
1040          * for the digits, or if all bits are used.
1041          */
1042         return -ENFILE;
1043 }
1044 
1045 /**
1046  *      dev_alloc_name - allocate a name for a device
1047  *      @dev: device
1048  *      @name: name format string
1049  *
1050  *      Passed a format string - eg "lt%d" it will try and find a suitable
1051  *      id. It scans list of devices to build up a free map, then chooses
1052  *      the first empty slot. The caller must hold the dev_base or rtnl lock
1053  *      while allocating the name and adding the device in order to avoid
1054  *      duplicates.
1055  *      Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1056  *      Returns the number of the unit assigned or a negative errno code.
1057  */
1058 
1059 int dev_alloc_name(struct net_device *dev, const char *name)
1060 {
1061         char buf[IFNAMSIZ];
1062         struct net *net;
1063         int ret;
1064 
1065         BUG_ON(!dev_net(dev));
1066         net = dev_net(dev);
1067         ret = __dev_alloc_name(net, name, buf);
1068         if (ret >= 0)
1069                 strlcpy(dev->name, buf, IFNAMSIZ);
1070         return ret;
1071 }
1072 EXPORT_SYMBOL(dev_alloc_name);
1073 
1074 static int dev_alloc_name_ns(struct net *net,
1075                              struct net_device *dev,
1076                              const char *name)
1077 {
1078         char buf[IFNAMSIZ];
1079         int ret;
1080 
1081         ret = __dev_alloc_name(net, name, buf);
1082         if (ret >= 0)
1083                 strlcpy(dev->name, buf, IFNAMSIZ);
1084         return ret;
1085 }
1086 
1087 static int dev_get_valid_name(struct net *net,
1088                               struct net_device *dev,
1089                               const char *name)
1090 {
1091         BUG_ON(!net);
1092 
1093         if (!dev_valid_name(name))
1094                 return -EINVAL;
1095 
1096         if (strchr(name, '%'))
1097                 return dev_alloc_name_ns(net, dev, name);
1098         else if (__dev_get_by_name(net, name))
1099                 return -EEXIST;
1100         else if (dev->name != name)
1101                 strlcpy(dev->name, name, IFNAMSIZ);
1102 
1103         return 0;
1104 }
1105 
1106 /**
1107  *      dev_change_name - change name of a device
1108  *      @dev: device
1109  *      @newname: name (or format string) must be at least IFNAMSIZ
1110  *
1111  *      Change name of a device, can pass format strings "eth%d".
1112  *      for wildcarding.
1113  */
1114 int dev_change_name(struct net_device *dev, const char *newname)
1115 {
1116         unsigned char old_assign_type;
1117         char oldname[IFNAMSIZ];
1118         int err = 0;
1119         int ret;
1120         struct net *net;
1121 
1122         ASSERT_RTNL();
1123         BUG_ON(!dev_net(dev));
1124 
1125         net = dev_net(dev);
1126         if (dev->flags & IFF_UP)
1127                 return -EBUSY;
1128 
1129         write_seqcount_begin(&devnet_rename_seq);
1130 
1131         if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
1132                 write_seqcount_end(&devnet_rename_seq);
1133                 return 0;
1134         }
1135 
1136         memcpy(oldname, dev->name, IFNAMSIZ);
1137 
1138         err = dev_get_valid_name(net, dev, newname);
1139         if (err < 0) {
1140                 write_seqcount_end(&devnet_rename_seq);
1141                 return err;
1142         }
1143 
1144         if (oldname[0] && !strchr(oldname, '%'))
1145                 netdev_info(dev, "renamed from %s\n", oldname);
1146 
1147         old_assign_type = dev->name_assign_type;
1148         dev->name_assign_type = NET_NAME_RENAMED;
1149 
1150 rollback:
1151         ret = device_rename(&dev->dev, dev->name);
1152         if (ret) {
1153                 memcpy(dev->name, oldname, IFNAMSIZ);
1154                 dev->name_assign_type = old_assign_type;
1155                 write_seqcount_end(&devnet_rename_seq);
1156                 return ret;
1157         }
1158 
1159         write_seqcount_end(&devnet_rename_seq);
1160 
1161         netdev_adjacent_rename_links(dev, oldname);
1162 
1163         write_lock_bh(&dev_base_lock);
1164         hlist_del_rcu(&dev->name_hlist);
1165         write_unlock_bh(&dev_base_lock);
1166 
1167         synchronize_rcu();
1168 
1169         write_lock_bh(&dev_base_lock);
1170         hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
1171         write_unlock_bh(&dev_base_lock);
1172 
1173         ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
1174         ret = notifier_to_errno(ret);
1175 
1176         if (ret) {
1177                 /* err >= 0 after dev_alloc_name() or stores the first errno */
1178                 if (err >= 0) {
1179                         err = ret;
1180                         write_seqcount_begin(&devnet_rename_seq);
1181                         memcpy(dev->name, oldname, IFNAMSIZ);
1182                         memcpy(oldname, newname, IFNAMSIZ);
1183                         dev->name_assign_type = old_assign_type;
1184                         old_assign_type = NET_NAME_RENAMED;
1185                         goto rollback;
1186                 } else {
1187                         pr_err("%s: name change rollback failed: %d\n",
1188                                dev->name, ret);
1189                 }
1190         }
1191 
1192         return err;
1193 }
1194 
1195 /**
1196  *      dev_set_alias - change ifalias of a device
1197  *      @dev: device
1198  *      @alias: name up to IFALIASZ
1199  *      @len: limit of bytes to copy from info
1200  *
1201  *      Set ifalias for a device,
1202  */
1203 int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1204 {
1205         char *new_ifalias;
1206 
1207         ASSERT_RTNL();
1208 
1209         if (len >= IFALIASZ)
1210                 return -EINVAL;
1211 
1212         if (!len) {
1213                 kfree(dev->ifalias);
1214                 dev->ifalias = NULL;
1215                 return 0;
1216         }
1217 
1218         new_ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
1219         if (!new_ifalias)
1220                 return -ENOMEM;
1221         dev->ifalias = new_ifalias;
1222 
1223         strlcpy(dev->ifalias, alias, len+1);
1224         return len;
1225 }
1226 
1227 
1228 /**
1229  *      netdev_features_change - device changes features
1230  *      @dev: device to cause notification
1231  *
1232  *      Called to indicate a device has changed features.
1233  */
1234 void netdev_features_change(struct net_device *dev)
1235 {
1236         call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
1237 }
1238 EXPORT_SYMBOL(netdev_features_change);
1239 
1240 /**
1241  *      netdev_state_change - device changes state
1242  *      @dev: device to cause notification
1243  *
1244  *      Called to indicate a device has changed state. This function calls
1245  *      the notifier chains for netdev_chain and sends a NEWLINK message
1246  *      to the routing socket.
1247  */
1248 void netdev_state_change(struct net_device *dev)
1249 {
1250         if (dev->flags & IFF_UP) {
1251                 struct netdev_notifier_change_info change_info;
1252 
1253                 change_info.flags_changed = 0;
1254                 call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
1255                                               &change_info.info);
1256                 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL);
1257         }
1258 }
1259 EXPORT_SYMBOL(netdev_state_change);
1260 
1261 /**
1262  *      netdev_notify_peers - notify network peers about existence of @dev
1263  *      @dev: network device
1264  *
1265  * Generate traffic such that interested network peers are aware of
1266  * @dev, such as by generating a gratuitous ARP. This may be used when
1267  * a device wants to inform the rest of the network about some sort of
1268  * reconfiguration such as a failover event or virtual machine
1269  * migration.
1270  */
1271 void netdev_notify_peers(struct net_device *dev)
1272 {
1273         rtnl_lock();
1274         call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1275         rtnl_unlock();
1276 }
1277 EXPORT_SYMBOL(netdev_notify_peers);
1278 
1279 static int __dev_open(struct net_device *dev)
1280 {
1281         const struct net_device_ops *ops = dev->netdev_ops;
1282         int ret;
1283 
1284         ASSERT_RTNL();
1285 
1286         if (!netif_device_present(dev))
1287                 return -ENODEV;
1288 
1289         /* Block netpoll from trying to do any rx path servicing.
1290          * If we don't do this there is a chance ndo_poll_controller
1291          * or ndo_poll may be running while we open the device
1292          */
1293         netpoll_poll_disable(dev);
1294 
1295         ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1296         ret = notifier_to_errno(ret);
1297         if (ret)
1298                 return ret;
1299 
1300         set_bit(__LINK_STATE_START, &dev->state);
1301 
1302         if (ops->ndo_validate_addr)
1303                 ret = ops->ndo_validate_addr(dev);
1304 
1305         if (!ret && ops->ndo_open)
1306                 ret = ops->ndo_open(dev);
1307 
1308         netpoll_poll_enable(dev);
1309 
1310         if (ret)
1311                 clear_bit(__LINK_STATE_START, &dev->state);
1312         else {
1313                 dev->flags |= IFF_UP;
1314                 dev_set_rx_mode(dev);
1315                 dev_activate(dev);
1316                 add_device_randomness(dev->dev_addr, dev->addr_len);
1317         }
1318 
1319         return ret;
1320 }
1321 
1322 /**
1323  *      dev_open        - prepare an interface for use.
1324  *      @dev:   device to open
1325  *
1326  *      Takes a device from down to up state. The device's private open
1327  *      function is invoked and then the multicast lists are loaded. Finally
1328  *      the device is moved into the up state and a %NETDEV_UP message is
1329  *      sent to the netdev notifier chain.
1330  *
1331  *      Calling this function on an active interface is a nop. On a failure
1332  *      a negative errno code is returned.
1333  */
1334 int dev_open(struct net_device *dev)
1335 {
1336         int ret;
1337 
1338         if (dev->flags & IFF_UP)
1339                 return 0;
1340 
1341         ret = __dev_open(dev);
1342         if (ret < 0)
1343                 return ret;
1344 
1345         rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1346         call_netdevice_notifiers(NETDEV_UP, dev);
1347 
1348         return ret;
1349 }
1350 EXPORT_SYMBOL(dev_open);
1351 
1352 static int __dev_close_many(struct list_head *head)
1353 {
1354         struct net_device *dev;
1355 
1356         ASSERT_RTNL();
1357         might_sleep();
1358 
1359         list_for_each_entry(dev, head, close_list) {
1360                 /* Temporarily disable netpoll until the interface is down */
1361                 netpoll_poll_disable(dev);
1362 
1363                 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1364 
1365                 clear_bit(__LINK_STATE_START, &dev->state);
1366 
1367                 /* Synchronize to scheduled poll. We cannot touch poll list, it
1368                  * can be even on different cpu. So just clear netif_running().
1369                  *
1370                  * dev->stop() will invoke napi_disable() on all of it's
1371                  * napi_struct instances on this device.
1372                  */
1373                 smp_mb__after_atomic(); /* Commit netif_running(). */
1374         }
1375 
1376         dev_deactivate_many(head);
1377 
1378         list_for_each_entry(dev, head, close_list) {
1379                 const struct net_device_ops *ops = dev->netdev_ops;
1380 
1381                 /*
1382                  *      Call the device specific close. This cannot fail.
1383                  *      Only if device is UP
1384                  *
1385                  *      We allow it to be called even after a DETACH hot-plug
1386                  *      event.
1387                  */
1388                 if (ops->ndo_stop)
1389                         ops->ndo_stop(dev);
1390 
1391                 dev->flags &= ~IFF_UP;
1392                 netpoll_poll_enable(dev);
1393         }
1394 
1395         return 0;
1396 }
1397 
1398 static int __dev_close(struct net_device *dev)
1399 {
1400         int retval;
1401         LIST_HEAD(single);
1402 
1403         list_add(&dev->close_list, &single);
1404         retval = __dev_close_many(&single);
1405         list_del(&single);
1406 
1407         return retval;
1408 }
1409 
1410 int dev_close_many(struct list_head *head, bool unlink)
1411 {
1412         struct net_device *dev, *tmp;
1413 
1414         /* Remove the devices that don't need to be closed */
1415         list_for_each_entry_safe(dev, tmp, head, close_list)
1416                 if (!(dev->flags & IFF_UP))
1417                         list_del_init(&dev->close_list);
1418 
1419         __dev_close_many(head);
1420 
1421         list_for_each_entry_safe(dev, tmp, head, close_list) {
1422                 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1423                 call_netdevice_notifiers(NETDEV_DOWN, dev);
1424                 if (unlink)
1425                         list_del_init(&dev->close_list);
1426         }
1427 
1428         return 0;
1429 }
1430 EXPORT_SYMBOL(dev_close_many);
1431 
1432 /**
1433  *      dev_close - shutdown an interface.
1434  *      @dev: device to shutdown
1435  *
1436  *      This function moves an active device into down state. A
1437  *      %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1438  *      is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1439  *      chain.
1440  */
1441 int dev_close(struct net_device *dev)
1442 {
1443         if (dev->flags & IFF_UP) {
1444                 LIST_HEAD(single);
1445 
1446                 list_add(&dev->close_list, &single);
1447                 dev_close_many(&single, true);
1448                 list_del(&single);
1449         }
1450         return 0;
1451 }
1452 EXPORT_SYMBOL(dev_close);
1453 
1454 
1455 /**
1456  *      dev_disable_lro - disable Large Receive Offload on a device
1457  *      @dev: device
1458  *
1459  *      Disable Large Receive Offload (LRO) on a net device.  Must be
1460  *      called under RTNL.  This is needed if received packets may be
1461  *      forwarded to another interface.
1462  */
1463 void dev_disable_lro(struct net_device *dev)
1464 {
1465         struct net_device *lower_dev;
1466         struct list_head *iter;
1467 
1468         dev->wanted_features &= ~NETIF_F_LRO;
1469         netdev_update_features(dev);
1470 
1471         if (unlikely(dev->features & NETIF_F_LRO))
1472                 netdev_WARN(dev, "failed to disable LRO!\n");
1473 
1474         netdev_for_each_lower_dev(dev, lower_dev, iter)
1475                 dev_disable_lro(lower_dev);
1476 }
1477 EXPORT_SYMBOL(dev_disable_lro);
1478 
1479 static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1480                                    struct net_device *dev)
1481 {
1482         struct netdev_notifier_info info;
1483 
1484         netdev_notifier_info_init(&info, dev);
1485         return nb->notifier_call(nb, val, &info);
1486 }
1487 
1488 static int dev_boot_phase = 1;
1489 
1490 /**
1491  *      register_netdevice_notifier - register a network notifier block
1492  *      @nb: notifier
1493  *
1494  *      Register a notifier to be called when network device events occur.
1495  *      The notifier passed is linked into the kernel structures and must
1496  *      not be reused until it has been unregistered. A negative errno code
1497  *      is returned on a failure.
1498  *
1499  *      When registered all registration and up events are replayed
1500  *      to the new notifier to allow device to have a race free
1501  *      view of the network device list.
1502  */
1503 
1504 int register_netdevice_notifier(struct notifier_block *nb)
1505 {
1506         struct net_device *dev;
1507         struct net_device *last;
1508         struct net *net;
1509         int err;
1510 
1511         rtnl_lock();
1512         err = raw_notifier_chain_register(&netdev_chain, nb);
1513         if (err)
1514                 goto unlock;
1515         if (dev_boot_phase)
1516                 goto unlock;
1517         for_each_net(net) {
1518                 for_each_netdev(net, dev) {
1519                         err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
1520                         err = notifier_to_errno(err);
1521                         if (err)
1522                                 goto rollback;
1523 
1524                         if (!(dev->flags & IFF_UP))
1525                                 continue;
1526 
1527                         call_netdevice_notifier(nb, NETDEV_UP, dev);
1528                 }
1529         }
1530 
1531 unlock:
1532         rtnl_unlock();
1533         return err;
1534 
1535 rollback:
1536         last = dev;
1537         for_each_net(net) {
1538                 for_each_netdev(net, dev) {
1539                         if (dev == last)
1540                                 goto outroll;
1541 
1542                         if (dev->flags & IFF_UP) {
1543                                 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1544                                                         dev);
1545                                 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1546                         }
1547                         call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1548                 }
1549         }
1550 
1551 outroll:
1552         raw_notifier_chain_unregister(&netdev_chain, nb);
1553         goto unlock;
1554 }
1555 EXPORT_SYMBOL(register_netdevice_notifier);
1556 
1557 /**
1558  *      unregister_netdevice_notifier - unregister a network notifier block
1559  *      @nb: notifier
1560  *
1561  *      Unregister a notifier previously registered by
1562  *      register_netdevice_notifier(). The notifier is unlinked into the
1563  *      kernel structures and may then be reused. A negative errno code
1564  *      is returned on a failure.
1565  *
1566  *      After unregistering unregister and down device events are synthesized
1567  *      for all devices on the device list to the removed notifier to remove
1568  *      the need for special case cleanup code.
1569  */
1570 
1571 int unregister_netdevice_notifier(struct notifier_block *nb)
1572 {
1573         struct net_device *dev;
1574         struct net *net;
1575         int err;
1576 
1577         rtnl_lock();
1578         err = raw_notifier_chain_unregister(&netdev_chain, nb);
1579         if (err)
1580                 goto unlock;
1581 
1582         for_each_net(net) {
1583                 for_each_netdev(net, dev) {
1584                         if (dev->flags & IFF_UP) {
1585                                 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1586                                                         dev);
1587                                 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1588                         }
1589                         call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1590                 }
1591         }
1592 unlock:
1593         rtnl_unlock();
1594         return err;
1595 }
1596 EXPORT_SYMBOL(unregister_netdevice_notifier);
1597 
1598 /**
1599  *      call_netdevice_notifiers_info - call all network notifier blocks
1600  *      @val: value passed unmodified to notifier function
1601  *      @dev: net_device pointer passed unmodified to notifier function
1602  *      @info: notifier information data
1603  *
1604  *      Call all network notifier blocks.  Parameters and return value
1605  *      are as for raw_notifier_call_chain().
1606  */
1607 
1608 static int call_netdevice_notifiers_info(unsigned long val,
1609                                          struct net_device *dev,
1610                                          struct netdev_notifier_info *info)
1611 {
1612         ASSERT_RTNL();
1613         netdev_notifier_info_init(info, dev);
1614         return raw_notifier_call_chain(&netdev_chain, val, info);
1615 }
1616 
1617 /**
1618  *      call_netdevice_notifiers - call all network notifier blocks
1619  *      @val: value passed unmodified to notifier function
1620  *      @dev: net_device pointer passed unmodified to notifier function
1621  *
1622  *      Call all network notifier blocks.  Parameters and return value
1623  *      are as for raw_notifier_call_chain().
1624  */
1625 
1626 int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
1627 {
1628         struct netdev_notifier_info info;
1629 
1630         return call_netdevice_notifiers_info(val, dev, &info);
1631 }
1632 EXPORT_SYMBOL(call_netdevice_notifiers);
1633 
1634 #ifdef CONFIG_NET_INGRESS
1635 static struct static_key ingress_needed __read_mostly;
1636 
1637 void net_inc_ingress_queue(void)
1638 {
1639         static_key_slow_inc(&ingress_needed);
1640 }
1641 EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
1642 
1643 void net_dec_ingress_queue(void)
1644 {
1645         static_key_slow_dec(&ingress_needed);
1646 }
1647 EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
1648 #endif
1649 
1650 static struct static_key netstamp_needed __read_mostly;
1651 #ifdef HAVE_JUMP_LABEL
1652 /* We are not allowed to call static_key_slow_dec() from irq context
1653  * If net_disable_timestamp() is called from irq context, defer the
1654  * static_key_slow_dec() calls.
1655  */
1656 static atomic_t netstamp_needed_deferred;
1657 #endif
1658 
1659 void net_enable_timestamp(void)
1660 {
1661 #ifdef HAVE_JUMP_LABEL
1662         int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
1663 
1664         if (deferred) {
1665                 while (--deferred)
1666                         static_key_slow_dec(&netstamp_needed);
1667                 return;
1668         }
1669 #endif
1670         static_key_slow_inc(&netstamp_needed);
1671 }
1672 EXPORT_SYMBOL(net_enable_timestamp);
1673 
1674 void net_disable_timestamp(void)
1675 {
1676 #ifdef HAVE_JUMP_LABEL
1677         if (in_interrupt()) {
1678                 atomic_inc(&netstamp_needed_deferred);
1679                 return;
1680         }
1681 #endif
1682         static_key_slow_dec(&netstamp_needed);
1683 }
1684 EXPORT_SYMBOL(net_disable_timestamp);
1685 
1686 static inline void net_timestamp_set(struct sk_buff *skb)
1687 {
1688         skb->tstamp.tv64 = 0;
1689         if (static_key_false(&netstamp_needed))
1690                 __net_timestamp(skb);
1691 }
1692 
1693 #define net_timestamp_check(COND, SKB)                  \
1694         if (static_key_false(&netstamp_needed)) {               \
1695                 if ((COND) && !(SKB)->tstamp.tv64)      \
1696                         __net_timestamp(SKB);           \
1697         }                                               \
1698 
1699 bool is_skb_forwardable(struct net_device *dev, struct sk_buff *skb)
1700 {
1701         unsigned int len;
1702 
1703         if (!(dev->flags & IFF_UP))
1704                 return false;
1705 
1706         len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
1707         if (skb->len <= len)
1708                 return true;
1709 
1710         /* if TSO is enabled, we don't care about the length as the packet
1711          * could be forwarded without being segmented before
1712          */
1713         if (skb_is_gso(skb))
1714                 return true;
1715 
1716         return false;
1717 }
1718 EXPORT_SYMBOL_GPL(is_skb_forwardable);
1719 
1720 int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1721 {
1722         if (skb_orphan_frags(skb, GFP_ATOMIC) ||
1723             unlikely(!is_skb_forwardable(dev, skb))) {
1724                 atomic_long_inc(&dev->rx_dropped);
1725                 kfree_skb(skb);
1726                 return NET_RX_DROP;
1727         }
1728 
1729         skb_scrub_packet(skb, true);
1730         skb->priority = 0;
1731         skb->protocol = eth_type_trans(skb, dev);
1732         skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
1733 
1734         return 0;
1735 }
1736 EXPORT_SYMBOL_GPL(__dev_forward_skb);
1737 
1738 /**
1739  * dev_forward_skb - loopback an skb to another netif
1740  *
1741  * @dev: destination network device
1742  * @skb: buffer to forward
1743  *
1744  * return values:
1745  *      NET_RX_SUCCESS  (no congestion)
1746  *      NET_RX_DROP     (packet was dropped, but freed)
1747  *
1748  * dev_forward_skb can be used for injecting an skb from the
1749  * start_xmit function of one device into the receive queue
1750  * of another device.
1751  *
1752  * The receiving device may be in another namespace, so
1753  * we have to clear all information in the skb that could
1754  * impact namespace isolation.
1755  */
1756 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1757 {
1758         return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
1759 }
1760 EXPORT_SYMBOL_GPL(dev_forward_skb);
1761 
1762 static inline int deliver_skb(struct sk_buff *skb,
1763                               struct packet_type *pt_prev,
1764                               struct net_device *orig_dev)
1765 {
1766         if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
1767                 return -ENOMEM;
1768         atomic_inc(&skb->users);
1769         return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1770 }
1771 
1772 static inline void deliver_ptype_list_skb(struct sk_buff *skb,
1773                                           struct packet_type **pt,
1774                                           struct net_device *orig_dev,
1775                                           __be16 type,
1776                                           struct list_head *ptype_list)
1777 {
1778         struct packet_type *ptype, *pt_prev = *pt;
1779 
1780         list_for_each_entry_rcu(ptype, ptype_list, list) {
1781                 if (ptype->type != type)
1782                         continue;
1783                 if (pt_prev)
1784                         deliver_skb(skb, pt_prev, orig_dev);
1785                 pt_prev = ptype;
1786         }
1787         *pt = pt_prev;
1788 }
1789 
1790 static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
1791 {
1792         if (!ptype->af_packet_priv || !skb->sk)
1793                 return false;
1794 
1795         if (ptype->id_match)
1796                 return ptype->id_match(ptype, skb->sk);
1797         else if ((struct sock *)ptype->af_packet_priv == skb->sk)
1798                 return true;
1799 
1800         return false;
1801 }
1802 
1803 /*
1804  *      Support routine. Sends outgoing frames to any network
1805  *      taps currently in use.
1806  */
1807 
1808 static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
1809 {
1810         struct packet_type *ptype;
1811         struct sk_buff *skb2 = NULL;
1812         struct packet_type *pt_prev = NULL;
1813         struct list_head *ptype_list = &ptype_all;
1814 
1815         rcu_read_lock();
1816 again:
1817         list_for_each_entry_rcu(ptype, ptype_list, list) {
1818                 /* Never send packets back to the socket
1819                  * they originated from - MvS (miquels@drinkel.ow.org)
1820                  */
1821                 if (skb_loop_sk(ptype, skb))
1822                         continue;
1823 
1824                 if (pt_prev) {
1825                         deliver_skb(skb2, pt_prev, skb->dev);
1826                         pt_prev = ptype;
1827                         continue;
1828                 }
1829 
1830                 /* need to clone skb, done only once */
1831                 skb2 = skb_clone(skb, GFP_ATOMIC);
1832                 if (!skb2)
1833                         goto out_unlock;
1834 
1835                 net_timestamp_set(skb2);
1836 
1837                 /* skb->nh should be correctly
1838                  * set by sender, so that the second statement is
1839                  * just protection against buggy protocols.
1840                  */
1841                 skb_reset_mac_header(skb2);
1842 
1843                 if (skb_network_header(skb2) < skb2->data ||
1844                     skb_network_header(skb2) > skb_tail_pointer(skb2)) {
1845                         net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
1846                                              ntohs(skb2->protocol),
1847                                              dev->name);
1848                         skb_reset_network_header(skb2);
1849                 }
1850 
1851                 skb2->transport_header = skb2->network_header;
1852                 skb2->pkt_type = PACKET_OUTGOING;
1853                 pt_prev = ptype;
1854         }
1855 
1856         if (ptype_list == &ptype_all) {
1857                 ptype_list = &dev->ptype_all;
1858                 goto again;
1859         }
1860 out_unlock:
1861         if (pt_prev)
1862                 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
1863         rcu_read_unlock();
1864 }
1865 
1866 /**
1867  * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
1868  * @dev: Network device
1869  * @txq: number of queues available
1870  *
1871  * If real_num_tx_queues is changed the tc mappings may no longer be
1872  * valid. To resolve this verify the tc mapping remains valid and if
1873  * not NULL the mapping. With no priorities mapping to this
1874  * offset/count pair it will no longer be used. In the worst case TC0
1875  * is invalid nothing can be done so disable priority mappings. If is
1876  * expected that drivers will fix this mapping if they can before
1877  * calling netif_set_real_num_tx_queues.
1878  */
1879 static void netif_setup_tc(struct net_device *dev, unsigned int txq)
1880 {
1881         int i;
1882         struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
1883 
1884         /* If TC0 is invalidated disable TC mapping */
1885         if (tc->offset + tc->count > txq) {
1886                 pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
1887                 dev->num_tc = 0;
1888                 return;
1889         }
1890 
1891         /* Invalidated prio to tc mappings set to TC0 */
1892         for (i = 1; i < TC_BITMASK + 1; i++) {
1893                 int q = netdev_get_prio_tc_map(dev, i);
1894 
1895                 tc = &dev->tc_to_txq[q];
1896                 if (tc->offset + tc->count > txq) {
1897                         pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
1898                                 i, q);
1899                         netdev_set_prio_tc_map(dev, i, 0);
1900                 }
1901         }
1902 }
1903 
1904 #ifdef CONFIG_XPS
1905 static DEFINE_MUTEX(xps_map_mutex);
1906 #define xmap_dereference(P)             \
1907         rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
1908 
1909 static struct xps_map *remove_xps_queue(struct xps_dev_maps *dev_maps,
1910                                         int cpu, u16 index)
1911 {
1912         struct xps_map *map = NULL;
1913         int pos;
1914 
1915         if (dev_maps)
1916                 map = xmap_dereference(dev_maps->cpu_map[cpu]);
1917 
1918         for (pos = 0; map && pos < map->len; pos++) {
1919                 if (map->queues[pos] == index) {
1920                         if (map->len > 1) {
1921                                 map->queues[pos] = map->queues[--map->len];
1922                         } else {
1923                                 RCU_INIT_POINTER(dev_maps->cpu_map[cpu], NULL);
1924                                 kfree_rcu(map, rcu);
1925                                 map = NULL;
1926                         }
1927                         break;
1928                 }
1929         }
1930 
1931         return map;
1932 }
1933 
1934 static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
1935 {
1936         struct xps_dev_maps *dev_maps;
1937         int cpu, i;
1938         bool active = false;
1939 
1940         mutex_lock(&xps_map_mutex);
1941         dev_maps = xmap_dereference(dev->xps_maps);
1942 
1943         if (!dev_maps)
1944                 goto out_no_maps;
1945 
1946         for_each_possible_cpu(cpu) {
1947                 for (i = index; i < dev->num_tx_queues; i++) {
1948                         if (!remove_xps_queue(dev_maps, cpu, i))
1949                                 break;
1950                 }
1951                 if (i == dev->num_tx_queues)
1952                         active = true;
1953         }
1954 
1955         if (!active) {
1956                 RCU_INIT_POINTER(dev->xps_maps, NULL);
1957                 kfree_rcu(dev_maps, rcu);
1958         }
1959 
1960         for (i = index; i < dev->num_tx_queues; i++)
1961                 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i),
1962                                              NUMA_NO_NODE);
1963 
1964 out_no_maps:
1965         mutex_unlock(&xps_map_mutex);
1966 }
1967 
1968 static struct xps_map *expand_xps_map(struct xps_map *map,
1969                                       int cpu, u16 index)
1970 {
1971         struct xps_map *new_map;
1972         int alloc_len = XPS_MIN_MAP_ALLOC;
1973         int i, pos;
1974 
1975         for (pos = 0; map && pos < map->len; pos++) {
1976                 if (map->queues[pos] != index)
1977                         continue;
1978                 return map;
1979         }
1980 
1981         /* Need to add queue to this CPU's existing map */
1982         if (map) {
1983                 if (pos < map->alloc_len)
1984                         return map;
1985 
1986                 alloc_len = map->alloc_len * 2;
1987         }
1988 
1989         /* Need to allocate new map to store queue on this CPU's map */
1990         new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
1991                                cpu_to_node(cpu));
1992         if (!new_map)
1993                 return NULL;
1994 
1995         for (i = 0; i < pos; i++)
1996                 new_map->queues[i] = map->queues[i];
1997         new_map->alloc_len = alloc_len;
1998         new_map->len = pos;
1999 
2000         return new_map;
2001 }
2002 
2003 int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2004                         u16 index)
2005 {
2006         struct xps_dev_maps *dev_maps, *new_dev_maps = NULL;
2007         struct xps_map *map, *new_map;
2008         int maps_sz = max_t(unsigned int, XPS_DEV_MAPS_SIZE, L1_CACHE_BYTES);
2009         int cpu, numa_node_id = -2;
2010         bool active = false;
2011 
2012         mutex_lock(&xps_map_mutex);
2013 
2014         dev_maps = xmap_dereference(dev->xps_maps);
2015 
2016         /* allocate memory for queue storage */
2017         for_each_online_cpu(cpu) {
2018                 if (!cpumask_test_cpu(cpu, mask))
2019                         continue;
2020 
2021                 if (!new_dev_maps)
2022                         new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
2023                 if (!new_dev_maps) {
2024                         mutex_unlock(&xps_map_mutex);
2025                         return -ENOMEM;
2026                 }
2027 
2028                 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) :
2029                                  NULL;
2030 
2031                 map = expand_xps_map(map, cpu, index);
2032                 if (!map)
2033                         goto error;
2034 
2035                 RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map);
2036         }
2037 
2038         if (!new_dev_maps)
2039                 goto out_no_new_maps;
2040 
2041         for_each_possible_cpu(cpu) {
2042                 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) {
2043                         /* add queue to CPU maps */
2044                         int pos = 0;
2045 
2046                         map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
2047                         while ((pos < map->len) && (map->queues[pos] != index))
2048                                 pos++;
2049 
2050                         if (pos == map->len)
2051                                 map->queues[map->len++] = index;
2052 #ifdef CONFIG_NUMA
2053                         if (numa_node_id == -2)
2054                                 numa_node_id = cpu_to_node(cpu);
2055                         else if (numa_node_id != cpu_to_node(cpu))
2056                                 numa_node_id = -1;
2057 #endif
2058                 } else if (dev_maps) {
2059                         /* fill in the new device map from the old device map */
2060                         map = xmap_dereference(dev_maps->cpu_map[cpu]);
2061                         RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map);
2062                 }
2063 
2064         }
2065 
2066         rcu_assign_pointer(dev->xps_maps, new_dev_maps);
2067 
2068         /* Cleanup old maps */
2069         if (dev_maps) {
2070                 for_each_possible_cpu(cpu) {
2071                         new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
2072                         map = xmap_dereference(dev_maps->cpu_map[cpu]);
2073                         if (map && map != new_map)
2074                                 kfree_rcu(map, rcu);
2075                 }
2076 
2077                 kfree_rcu(dev_maps, rcu);
2078         }
2079 
2080         dev_maps = new_dev_maps;
2081         active = true;
2082 
2083 out_no_new_maps:
2084         /* update Tx queue numa node */
2085         netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2086                                      (numa_node_id >= 0) ? numa_node_id :
2087                                      NUMA_NO_NODE);
2088 
2089         if (!dev_maps)
2090                 goto out_no_maps;
2091 
2092         /* removes queue from unused CPUs */
2093         for_each_possible_cpu(cpu) {
2094                 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu))
2095                         continue;
2096 
2097                 if (remove_xps_queue(dev_maps, cpu, index))
2098                         active = true;
2099         }
2100 
2101         /* free map if not active */
2102         if (!active) {
2103                 RCU_INIT_POINTER(dev->xps_maps, NULL);
2104                 kfree_rcu(dev_maps, rcu);
2105         }
2106 
2107 out_no_maps:
2108         mutex_unlock(&xps_map_mutex);
2109 
2110         return 0;
2111 error:
2112         /* remove any maps that we added */
2113         for_each_possible_cpu(cpu) {
2114                 new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
2115                 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) :
2116                                  NULL;
2117                 if (new_map && new_map != map)
2118                         kfree(new_map);
2119         }
2120 
2121         mutex_unlock(&xps_map_mutex);
2122 
2123         kfree(new_dev_maps);
2124         return -ENOMEM;
2125 }
2126 EXPORT_SYMBOL(netif_set_xps_queue);
2127 
2128 #endif
2129 /*
2130  * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
2131  * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
2132  */
2133 int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
2134 {
2135         int rc;
2136 
2137         if (txq < 1 || txq > dev->num_tx_queues)
2138                 return -EINVAL;
2139 
2140         if (dev->reg_state == NETREG_REGISTERED ||
2141             dev->reg_state == NETREG_UNREGISTERING) {
2142                 ASSERT_RTNL();
2143 
2144                 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
2145                                                   txq);
2146                 if (rc)
2147                         return rc;
2148 
2149                 if (dev->num_tc)
2150                         netif_setup_tc(dev, txq);
2151 
2152                 if (txq < dev->real_num_tx_queues) {
2153                         qdisc_reset_all_tx_gt(dev, txq);
2154 #ifdef CONFIG_XPS
2155                         netif_reset_xps_queues_gt(dev, txq);
2156 #endif
2157                 }
2158         }
2159 
2160         dev->real_num_tx_queues = txq;
2161         return 0;
2162 }
2163 EXPORT_SYMBOL(netif_set_real_num_tx_queues);
2164 
2165 #ifdef CONFIG_SYSFS
2166 /**
2167  *      netif_set_real_num_rx_queues - set actual number of RX queues used
2168  *      @dev: Network device
2169  *      @rxq: Actual number of RX queues
2170  *
2171  *      This must be called either with the rtnl_lock held or before
2172  *      registration of the net device.  Returns 0 on success, or a
2173  *      negative error code.  If called before registration, it always
2174  *      succeeds.
2175  */
2176 int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
2177 {
2178         int rc;
2179 
2180         if (rxq < 1 || rxq > dev->num_rx_queues)
2181                 return -EINVAL;
2182 
2183         if (dev->reg_state == NETREG_REGISTERED) {
2184                 ASSERT_RTNL();
2185 
2186                 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
2187                                                   rxq);
2188                 if (rc)
2189                         return rc;
2190         }
2191 
2192         dev->real_num_rx_queues = rxq;
2193         return 0;
2194 }
2195 EXPORT_SYMBOL(netif_set_real_num_rx_queues);
2196 #endif
2197 
2198 /**
2199  * netif_get_num_default_rss_queues - default number of RSS queues
2200  *
2201  * This routine should set an upper limit on the number of RSS queues
2202  * used by default by multiqueue devices.
2203  */
2204 int netif_get_num_default_rss_queues(void)
2205 {
2206         return min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
2207 }
2208 EXPORT_SYMBOL(netif_get_num_default_rss_queues);
2209 
2210 static inline void __netif_reschedule(struct Qdisc *q)
2211 {
2212         struct softnet_data *sd;
2213         unsigned long flags;
2214 
2215         local_irq_save(flags);
2216         sd = this_cpu_ptr(&softnet_data);
2217         q->next_sched = NULL;
2218         *sd->output_queue_tailp = q;
2219         sd->output_queue_tailp = &q->next_sched;
2220         raise_softirq_irqoff(NET_TX_SOFTIRQ);
2221         local_irq_restore(flags);
2222 }
2223 
2224 void __netif_schedule(struct Qdisc *q)
2225 {
2226         if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
2227                 __netif_reschedule(q);
2228 }
2229 EXPORT_SYMBOL(__netif_schedule);
2230 
2231 struct dev_kfree_skb_cb {
2232         enum skb_free_reason reason;
2233 };
2234 
2235 static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
2236 {
2237         return (struct dev_kfree_skb_cb *)skb->cb;
2238 }
2239 
2240 void netif_schedule_queue(struct netdev_queue *txq)
2241 {
2242         rcu_read_lock();
2243         if (!(txq->state & QUEUE_STATE_ANY_XOFF)) {
2244                 struct Qdisc *q = rcu_dereference(txq->qdisc);
2245 
2246                 __netif_schedule(q);
2247         }
2248         rcu_read_unlock();
2249 }
2250 EXPORT_SYMBOL(netif_schedule_queue);
2251 
2252 /**
2253  *      netif_wake_subqueue - allow sending packets on subqueue
2254  *      @dev: network device
2255  *      @queue_index: sub queue index
2256  *
2257  * Resume individual transmit queue of a device with multiple transmit queues.
2258  */
2259 void netif_wake_subqueue(struct net_device *dev, u16 queue_index)
2260 {
2261         struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
2262 
2263         if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &txq->state)) {
2264                 struct Qdisc *q;
2265 
2266                 rcu_read_lock();
2267                 q = rcu_dereference(txq->qdisc);
2268                 __netif_schedule(q);
2269                 rcu_read_unlock();
2270         }
2271 }
2272 EXPORT_SYMBOL(netif_wake_subqueue);
2273 
2274 void netif_tx_wake_queue(struct netdev_queue *dev_queue)
2275 {
2276         if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
2277                 struct Qdisc *q;
2278 
2279                 rcu_read_lock();
2280                 q = rcu_dereference(dev_queue->qdisc);
2281                 __netif_schedule(q);
2282                 rcu_read_unlock();
2283         }
2284 }
2285 EXPORT_SYMBOL(netif_tx_wake_queue);
2286 
2287 void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason)
2288 {
2289         unsigned long flags;
2290 
2291         if (likely(atomic_read(&skb->users) == 1)) {
2292                 smp_rmb();
2293                 atomic_set(&skb->users, 0);
2294         } else if (likely(!atomic_dec_and_test(&skb->users))) {
2295                 return;
2296         }
2297         get_kfree_skb_cb(skb)->reason = reason;
2298         local_irq_save(flags);
2299         skb->next = __this_cpu_read(softnet_data.completion_queue);
2300         __this_cpu_write(softnet_data.completion_queue, skb);
2301         raise_softirq_irqoff(NET_TX_SOFTIRQ);
2302         local_irq_restore(flags);
2303 }
2304 EXPORT_SYMBOL(__dev_kfree_skb_irq);
2305 
2306 void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason)
2307 {
2308         if (in_irq() || irqs_disabled())
2309                 __dev_kfree_skb_irq(skb, reason);
2310         else
2311                 dev_kfree_skb(skb);
2312 }
2313 EXPORT_SYMBOL(__dev_kfree_skb_any);
2314 
2315 
2316 /**
2317  * netif_device_detach - mark device as removed
2318  * @dev: network device
2319  *
2320  * Mark device as removed from system and therefore no longer available.
2321  */
2322 void netif_device_detach(struct net_device *dev)
2323 {
2324         if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
2325             netif_running(dev)) {
2326                 netif_tx_stop_all_queues(dev);
2327         }
2328 }
2329 EXPORT_SYMBOL(netif_device_detach);
2330 
2331 /**
2332  * netif_device_attach - mark device as attached
2333  * @dev: network device
2334  *
2335  * Mark device as attached from system and restart if needed.
2336  */
2337 void netif_device_attach(struct net_device *dev)
2338 {
2339         if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
2340             netif_running(dev)) {
2341                 netif_tx_wake_all_queues(dev);
2342                 __netdev_watchdog_up(dev);
2343         }
2344 }
2345 EXPORT_SYMBOL(netif_device_attach);
2346 
2347 /*
2348  * Returns a Tx hash based on the given packet descriptor a Tx queues' number
2349  * to be used as a distribution range.
2350  */
2351 u16 __skb_tx_hash(const struct net_device *dev, struct sk_buff *skb,
2352                   unsigned int num_tx_queues)
2353 {
2354         u32 hash;
2355         u16 qoffset = 0;
2356         u16 qcount = num_tx_queues;
2357 
2358         if (skb_rx_queue_recorded(skb)) {
2359                 hash = skb_get_rx_queue(skb);
2360                 while (unlikely(hash >= num_tx_queues))
2361                         hash -= num_tx_queues;
2362                 return hash;
2363         }
2364 
2365         if (dev->num_tc) {
2366                 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
2367                 qoffset = dev->tc_to_txq[tc].offset;
2368                 qcount = dev->tc_to_txq[tc].count;
2369         }
2370 
2371         return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
2372 }
2373 EXPORT_SYMBOL(__skb_tx_hash);
2374 
2375 static void skb_warn_bad_offload(const struct sk_buff *skb)
2376 {
2377         static const netdev_features_t null_features = 0;
2378         struct net_device *dev = skb->dev;
2379         const char *driver = "";
2380 
2381         if (!net_ratelimit())
2382                 return;
2383 
2384         if (dev && dev->dev.parent)
2385                 driver = dev_driver_string(dev->dev.parent);
2386 
2387         WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d "
2388              "gso_type=%d ip_summed=%d\n",
2389              driver, dev ? &dev->features : &null_features,
2390              skb->sk ? &skb->sk->sk_route_caps : &null_features,
2391              skb->len, skb->data_len, skb_shinfo(skb)->gso_size,
2392              skb_shinfo(skb)->gso_type, skb->ip_summed);
2393 }
2394 
2395 /*
2396  * Invalidate hardware checksum when packet is to be mangled, and
2397  * complete checksum manually on outgoing path.
2398  */
2399 int skb_checksum_help(struct sk_buff *skb)
2400 {
2401         __wsum csum;
2402         int ret = 0, offset;
2403 
2404         if (skb->ip_summed == CHECKSUM_COMPLETE)
2405                 goto out_set_summed;
2406 
2407         if (unlikely(skb_shinfo(skb)->gso_size)) {
2408                 skb_warn_bad_offload(skb);
2409                 return -EINVAL;
2410         }
2411 
2412         /* Before computing a checksum, we should make sure no frag could
2413          * be modified by an external entity : checksum could be wrong.
2414          */
2415         if (skb_has_shared_frag(skb)) {
2416                 ret = __skb_linearize(skb);
2417                 if (ret)
2418                         goto out;
2419         }
2420 
2421         offset = skb_checksum_start_offset(skb);
2422         BUG_ON(offset >= skb_headlen(skb));
2423         csum = skb_checksum(skb, offset, skb->len - offset, 0);
2424 
2425         offset += skb->csum_offset;
2426         BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
2427 
2428         if (skb_cloned(skb) &&
2429             !skb_clone_writable(skb, offset + sizeof(__sum16))) {
2430                 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2431                 if (ret)
2432                         goto out;
2433         }
2434 
2435         *(__sum16 *)(skb->data + offset) = csum_fold(csum);
2436 out_set_summed:
2437         skb->ip_summed = CHECKSUM_NONE;
2438 out:
2439         return ret;
2440 }
2441 EXPORT_SYMBOL(skb_checksum_help);
2442 
2443 __be16 skb_network_protocol(struct sk_buff *skb, int *depth)
2444 {
2445         __be16 type = skb->protocol;
2446 
2447         /* Tunnel gso handlers can set protocol to ethernet. */
2448         if (type == htons(ETH_P_TEB)) {
2449                 struct ethhdr *eth;
2450 
2451                 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
2452                         return 0;
2453 
2454                 eth = (struct ethhdr *)skb_mac_header(skb);
2455                 type = eth->h_proto;
2456         }
2457 
2458         return __vlan_get_protocol(skb, type, depth);
2459 }
2460 
2461 /**
2462  *      skb_mac_gso_segment - mac layer segmentation handler.
2463  *      @skb: buffer to segment
2464  *      @features: features for the output path (see dev->features)
2465  */
2466 struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
2467                                     netdev_features_t features)
2468 {
2469         struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
2470         struct packet_offload *ptype;
2471         int vlan_depth = skb->mac_len;
2472         __be16 type = skb_network_protocol(skb, &vlan_depth);
2473 
2474         if (unlikely(!type))
2475                 return ERR_PTR(-EINVAL);
2476 
2477         __skb_pull(skb, vlan_depth);
2478 
2479         rcu_read_lock();
2480         list_for_each_entry_rcu(ptype, &offload_base, list) {
2481                 if (ptype->type == type && ptype->callbacks.gso_segment) {
2482                         segs = ptype->callbacks.gso_segment(skb, features);
2483                         break;
2484                 }
2485         }
2486         rcu_read_unlock();
2487 
2488         __skb_push(skb, skb->data - skb_mac_header(skb));
2489 
2490         return segs;
2491 }
2492 EXPORT_SYMBOL(skb_mac_gso_segment);
2493 
2494 
2495 /* openvswitch calls this on rx path, so we need a different check.
2496  */
2497 static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
2498 {
2499         if (tx_path)
2500                 return skb->ip_summed != CHECKSUM_PARTIAL;
2501         else
2502                 return skb->ip_summed == CHECKSUM_NONE;
2503 }
2504 
2505 /**
2506  *      __skb_gso_segment - Perform segmentation on skb.
2507  *      @skb: buffer to segment
2508  *      @features: features for the output path (see dev->features)
2509  *      @tx_path: whether it is called in TX path
2510  *
2511  *      This function segments the given skb and returns a list of segments.
2512  *
2513  *      It may return NULL if the skb requires no segmentation.  This is
2514  *      only possible when GSO is used for verifying header integrity.
2515  */
2516 struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
2517                                   netdev_features_t features, bool tx_path)
2518 {
2519         if (unlikely(skb_needs_check(skb, tx_path))) {
2520                 int err;
2521 
2522                 skb_warn_bad_offload(skb);
2523 
2524                 err = skb_cow_head(skb, 0);
2525                 if (err < 0)
2526                         return ERR_PTR(err);
2527         }
2528 
2529         SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
2530         SKB_GSO_CB(skb)->encap_level = 0;
2531 
2532         skb_reset_mac_header(skb);
2533         skb_reset_mac_len(skb);
2534 
2535         return skb_mac_gso_segment(skb, features);
2536 }
2537 EXPORT_SYMBOL(__skb_gso_segment);
2538 
2539 /* Take action when hardware reception checksum errors are detected. */
2540 #ifdef CONFIG_BUG
2541 void netdev_rx_csum_fault(struct net_device *dev)
2542 {
2543         if (net_ratelimit()) {
2544                 pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
2545                 dump_stack();
2546         }
2547 }
2548 EXPORT_SYMBOL(netdev_rx_csum_fault);
2549 #endif
2550 
2551 /* Actually, we should eliminate this check as soon as we know, that:
2552  * 1. IOMMU is present and allows to map all the memory.
2553  * 2. No high memory really exists on this machine.
2554  */
2555 
2556 static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
2557 {
2558 #ifdef CONFIG_HIGHMEM
2559         int i;
2560         if (!(dev->features & NETIF_F_HIGHDMA)) {
2561                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2562                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2563                         if (PageHighMem(skb_frag_page(frag)))
2564                                 return 1;
2565                 }
2566         }
2567 
2568         if (PCI_DMA_BUS_IS_PHYS) {
2569                 struct device *pdev = dev->dev.parent;
2570 
2571                 if (!pdev)
2572                         return 0;
2573                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2574                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2575                         dma_addr_t addr = page_to_phys(skb_frag_page(frag));
2576                         if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
2577                                 return 1;
2578                 }
2579         }
2580 #endif
2581         return 0;
2582 }
2583 
2584 /* If MPLS offload request, verify we are testing hardware MPLS features
2585  * instead of standard features for the netdev.
2586  */
2587 #if IS_ENABLED(CONFIG_NET_MPLS_GSO)
2588 static netdev_features_t net_mpls_features(struct sk_buff *skb,
2589                                            netdev_features_t features,
2590                                            __be16 type)
2591 {
2592         if (eth_p_mpls(type))
2593                 features &= skb->dev->mpls_features;
2594 
2595         return features;
2596 }
2597 #else
2598 static netdev_features_t net_mpls_features(struct sk_buff *skb,
2599                                            netdev_features_t features,
2600                                            __be16 type)
2601 {
2602         return features;
2603 }
2604 #endif
2605 
2606 static netdev_features_t harmonize_features(struct sk_buff *skb,
2607         netdev_features_t features)
2608 {
2609         int tmp;
2610         __be16 type;
2611 
2612         type = skb_network_protocol(skb, &tmp);
2613         features = net_mpls_features(skb, features, type);
2614 
2615         if (skb->ip_summed != CHECKSUM_NONE &&
2616             !can_checksum_protocol(features, type)) {
2617                 features &= ~NETIF_F_ALL_CSUM;
2618         } else if (illegal_highdma(skb->dev, skb)) {
2619                 features &= ~NETIF_F_SG;
2620         }
2621 
2622         return features;
2623 }
2624 
2625 netdev_features_t passthru_features_check(struct sk_buff *skb,
2626                                           struct net_device *dev,
2627                                           netdev_features_t features)
2628 {
2629         return features;
2630 }
2631 EXPORT_SYMBOL(passthru_features_check);
2632 
2633 static netdev_features_t dflt_features_check(const struct sk_buff *skb,
2634                                              struct net_device *dev,
2635                                              netdev_features_t features)
2636 {
2637         return vlan_features_check(skb, features);
2638 }
2639 
2640 netdev_features_t netif_skb_features(struct sk_buff *skb)
2641 {
2642         struct net_device *dev = skb->dev;
2643         netdev_features_t features = dev->features;
2644         u16 gso_segs = skb_shinfo(skb)->gso_segs;
2645 
2646         if (gso_segs > dev->gso_max_segs || gso_segs < dev->gso_min_segs)
2647                 features &= ~NETIF_F_GSO_MASK;
2648 
2649         /* If encapsulation offload request, verify we are testing
2650          * hardware encapsulation features instead of standard
2651          * features for the netdev
2652          */
2653         if (skb->encapsulation)
2654                 features &= dev->hw_enc_features;
2655 
2656         if (skb_vlan_tagged(skb))
2657                 features = netdev_intersect_features(features,
2658                                                      dev->vlan_features |
2659                                                      NETIF_F_HW_VLAN_CTAG_TX |
2660                                                      NETIF_F_HW_VLAN_STAG_TX);
2661 
2662         if (dev->netdev_ops->ndo_features_check)
2663                 features &= dev->netdev_ops->ndo_features_check(skb, dev,
2664                                                                 features);
2665         else
2666                 features &= dflt_features_check(skb, dev, features);
2667 
2668         return harmonize_features(skb, features);
2669 }
2670 EXPORT_SYMBOL(netif_skb_features);
2671 
2672 static int xmit_one(struct sk_buff *skb, struct net_device *dev,
2673                     struct netdev_queue *txq, bool more)
2674 {
2675         unsigned int len;
2676         int rc;
2677 
2678         if (!list_empty(&ptype_all) || !list_empty(&dev->ptype_all))
2679                 dev_queue_xmit_nit(skb, dev);
2680 
2681         len = skb->len;
2682         trace_net_dev_start_xmit(skb, dev);
2683         rc = netdev_start_xmit(skb, dev, txq, more);
2684         trace_net_dev_xmit(skb, rc, dev, len);
2685 
2686         return rc;
2687 }
2688 
2689 struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
2690                                     struct netdev_queue *txq, int *ret)
2691 {
2692         struct sk_buff *skb = first;
2693         int rc = NETDEV_TX_OK;
2694 
2695         while (skb) {
2696                 struct sk_buff *next = skb->next;
2697 
2698                 skb->next = NULL;
2699                 rc = xmit_one(skb, dev, txq, next != NULL);
2700                 if (unlikely(!dev_xmit_complete(rc))) {
2701                         skb->next = next;
2702                         goto out;
2703                 }
2704 
2705                 skb = next;
2706                 if (netif_xmit_stopped(txq) && skb) {
2707                         rc = NETDEV_TX_BUSY;
2708                         break;
2709                 }
2710         }
2711 
2712 out:
2713         *ret = rc;
2714         return skb;
2715 }
2716 
2717 static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
2718                                           netdev_features_t features)
2719 {
2720         if (skb_vlan_tag_present(skb) &&
2721             !vlan_hw_offload_capable(features, skb->vlan_proto))
2722                 skb = __vlan_hwaccel_push_inside(skb);
2723         return skb;
2724 }
2725 
2726 static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev)
2727 {
2728         netdev_features_t features;
2729 
2730         if (skb->next)
2731                 return skb;
2732 
2733         features = netif_skb_features(skb);
2734         skb = validate_xmit_vlan(skb, features);
2735         if (unlikely(!skb))
2736                 goto out_null;
2737 
2738         if (netif_needs_gso(skb, features)) {
2739                 struct sk_buff *segs;
2740 
2741                 segs = skb_gso_segment(skb, features);
2742                 if (IS_ERR(segs)) {
2743                         goto out_kfree_skb;
2744                 } else if (segs) {
2745                         consume_skb(skb);
2746                         skb = segs;
2747                 }
2748         } else {
2749                 if (skb_needs_linearize(skb, features) &&
2750                     __skb_linearize(skb))
2751                         goto out_kfree_skb;
2752 
2753                 /* If packet is not checksummed and device does not
2754                  * support checksumming for this protocol, complete
2755                  * checksumming here.
2756                  */
2757                 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2758                         if (skb->encapsulation)
2759                                 skb_set_inner_transport_header(skb,
2760                                                                skb_checksum_start_offset(skb));
2761                         else
2762                                 skb_set_transport_header(skb,
2763                                                          skb_checksum_start_offset(skb));
2764                         if (!(features & NETIF_F_ALL_CSUM) &&
2765                             skb_checksum_help(skb))
2766                                 goto out_kfree_skb;
2767                 }
2768         }
2769 
2770         return skb;
2771 
2772 out_kfree_skb:
2773         kfree_skb(skb);
2774 out_null:
2775         return NULL;
2776 }
2777 
2778 struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev)
2779 {
2780         struct sk_buff *next, *head = NULL, *tail;
2781 
2782         for (; skb != NULL; skb = next) {
2783                 next = skb->next;
2784                 skb->next = NULL;
2785 
2786                 /* in case skb wont be segmented, point to itself */
2787                 skb->prev = skb;
2788 
2789                 skb = validate_xmit_skb(skb, dev);
2790                 if (!skb)
2791                         continue;
2792 
2793                 if (!head)
2794                         head = skb;
2795                 else
2796                         tail->next = skb;
2797                 /* If skb was segmented, skb->prev points to
2798                  * the last segment. If not, it still contains skb.
2799                  */
2800                 tail = skb->prev;
2801         }
2802         return head;
2803 }
2804 
2805 static void qdisc_pkt_len_init(struct sk_buff *skb)
2806 {
2807         const struct skb_shared_info *shinfo = skb_shinfo(skb);
2808 
2809         qdisc_skb_cb(skb)->pkt_len = skb->len;
2810 
2811         /* To get more precise estimation of bytes sent on wire,
2812          * we add to pkt_len the headers size of all segments
2813          */
2814         if (shinfo->gso_size)  {
2815                 unsigned int hdr_len;
2816                 u16 gso_segs = shinfo->gso_segs;
2817 
2818                 /* mac layer + network layer */
2819                 hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
2820 
2821                 /* + transport layer */
2822                 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
2823                         hdr_len += tcp_hdrlen(skb);
2824                 else
2825                         hdr_len += sizeof(struct udphdr);
2826 
2827                 if (shinfo->gso_type & SKB_GSO_DODGY)
2828                         gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
2829                                                 shinfo->gso_size);
2830 
2831                 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
2832         }
2833 }
2834 
2835 static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
2836                                  struct net_device *dev,
2837                                  struct netdev_queue *txq)
2838 {
2839         spinlock_t *root_lock = qdisc_lock(q);
2840         bool contended;
2841         int rc;
2842 
2843         qdisc_pkt_len_init(skb);
2844         qdisc_calculate_pkt_len(skb, q);
2845         /*
2846          * Heuristic to force contended enqueues to serialize on a
2847          * separate lock before trying to get qdisc main lock.
2848          * This permits __QDISC___STATE_RUNNING owner to get the lock more
2849          * often and dequeue packets faster.
2850          */
2851         contended = qdisc_is_running(q);
2852         if (unlikely(contended))
2853                 spin_lock(&q->busylock);
2854 
2855         spin_lock(root_lock);
2856         if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
2857                 kfree_skb(skb);
2858                 rc = NET_XMIT_DROP;
2859         } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
2860                    qdisc_run_begin(q)) {
2861                 /*
2862                  * This is a work-conserving queue; there are no old skbs
2863                  * waiting to be sent out; and the qdisc is not running -
2864                  * xmit the skb directly.
2865                  */
2866 
2867                 qdisc_bstats_update(q, skb);
2868 
2869                 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
2870                         if (unlikely(contended)) {
2871                                 spin_unlock(&q->busylock);
2872                                 contended = false;
2873                         }
2874                         __qdisc_run(q);
2875                 } else
2876                         qdisc_run_end(q);
2877 
2878                 rc = NET_XMIT_SUCCESS;
2879         } else {
2880                 rc = q->enqueue(skb, q) & NET_XMIT_MASK;
2881                 if (qdisc_run_begin(q)) {
2882                         if (unlikely(contended)) {
2883                                 spin_unlock(&q->busylock);
2884                                 contended = false;
2885                         }
2886                         __qdisc_run(q);
2887                 }
2888         }
2889         spin_unlock(root_lock);
2890         if (unlikely(contended))
2891                 spin_unlock(&q->busylock);
2892         return rc;
2893 }
2894 
2895 #if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
2896 static void skb_update_prio(struct sk_buff *skb)
2897 {
2898         struct netprio_map *map = rcu_dereference_bh(skb->dev->priomap);
2899 
2900         if (!skb->priority && skb->sk && map) {
2901                 unsigned int prioidx = skb->sk->sk_cgrp_prioidx;
2902 
2903                 if (prioidx < map->priomap_len)
2904                         skb->priority = map->priomap[prioidx];
2905         }
2906 }
2907 #else
2908 #define skb_update_prio(skb)
2909 #endif
2910 
2911 DEFINE_PER_CPU(int, xmit_recursion);
2912 EXPORT_SYMBOL(xmit_recursion);
2913 
2914 #define RECURSION_LIMIT 10
2915 
2916 /**
2917  *      dev_loopback_xmit - loop back @skb
2918  *      @skb: buffer to transmit
2919  */
2920 int dev_loopback_xmit(struct sock *sk, struct sk_buff *skb)
2921 {
2922         skb_reset_mac_header(skb);
2923         __skb_pull(skb, skb_network_offset(skb));
2924         skb->pkt_type = PACKET_LOOPBACK;
2925         skb->ip_summed = CHECKSUM_UNNECESSARY;
2926         WARN_ON(!skb_dst(skb));
2927         skb_dst_force(skb);
2928         netif_rx_ni(skb);
2929         return 0;
2930 }
2931 EXPORT_SYMBOL(dev_loopback_xmit);
2932 
2933 static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb)
2934 {
2935 #ifdef CONFIG_XPS
2936         struct xps_dev_maps *dev_maps;
2937         struct xps_map *map;
2938         int queue_index = -1;
2939 
2940         rcu_read_lock();
2941         dev_maps = rcu_dereference(dev->xps_maps);
2942         if (dev_maps) {
2943                 map = rcu_dereference(
2944                     dev_maps->cpu_map[skb->sender_cpu - 1]);
2945                 if (map) {
2946                         if (map->len == 1)
2947                                 queue_index = map->queues[0];
2948                         else
2949                                 queue_index = map->queues[reciprocal_scale(skb_get_hash(skb),
2950                                                                            map->len)];
2951                         if (unlikely(queue_index >= dev->real_num_tx_queues))
2952                                 queue_index = -1;
2953                 }
2954         }
2955         rcu_read_unlock();
2956 
2957         return queue_index;
2958 #else
2959         return -1;
2960 #endif
2961 }
2962 
2963 static u16 __netdev_pick_tx(struct net_device *dev, struct sk_buff *skb)
2964 {
2965         struct sock *sk = skb->sk;
2966         int queue_index = sk_tx_queue_get(sk);
2967 
2968         if (queue_index < 0 || skb->ooo_okay ||
2969             queue_index >= dev->real_num_tx_queues) {
2970                 int new_index = get_xps_queue(dev, skb);
2971                 if (new_index < 0)
2972                         new_index = skb_tx_hash(dev, skb);
2973 
2974                 if (queue_index != new_index && sk &&
2975                     rcu_access_pointer(sk->sk_dst_cache))
2976                         sk_tx_queue_set(sk, new_index);
2977 
2978                 queue_index = new_index;
2979         }
2980 
2981         return queue_index;
2982 }
2983 
2984 struct netdev_queue *netdev_pick_tx(struct net_device *dev,
2985                                     struct sk_buff *skb,
2986                                     void *accel_priv)
2987 {
2988         int queue_index = 0;
2989 
2990 #ifdef CONFIG_XPS
2991         if (skb->sender_cpu == 0)
2992                 skb->sender_cpu = raw_smp_processor_id() + 1;
2993 #endif
2994 
2995         if (dev->real_num_tx_queues != 1) {
2996                 const struct net_device_ops *ops = dev->netdev_ops;
2997                 if (ops->ndo_select_queue)
2998                         queue_index = ops->ndo_select_queue(dev, skb, accel_priv,
2999                                                             __netdev_pick_tx);
3000                 else
3001                         queue_index = __netdev_pick_tx(dev, skb);
3002 
3003                 if (!accel_priv)
3004                         queue_index = netdev_cap_txqueue(dev, queue_index);
3005         }
3006 
3007         skb_set_queue_mapping(skb, queue_index);
3008         return netdev_get_tx_queue(dev, queue_index);
3009 }
3010 
3011 /**
3012  *      __dev_queue_xmit - transmit a buffer
3013  *      @skb: buffer to transmit
3014  *      @accel_priv: private data used for L2 forwarding offload
3015  *
3016  *      Queue a buffer for transmission to a network device. The caller must
3017  *      have set the device and priority and built the buffer before calling
3018  *      this function. The function can be called from an interrupt.
3019  *
3020  *      A negative errno code is returned on a failure. A success does not
3021  *      guarantee the frame will be transmitted as it may be dropped due
3022  *      to congestion or traffic shaping.
3023  *
3024  * -----------------------------------------------------------------------------------
3025  *      I notice this method can also return errors from the queue disciplines,
3026  *      including NET_XMIT_DROP, which is a positive value.  So, errors can also
3027  *      be positive.
3028  *
3029  *      Regardless of the return value, the skb is consumed, so it is currently
3030  *      difficult to retry a send to this method.  (You can bump the ref count
3031  *      before sending to hold a reference for retry if you are careful.)
3032  *
3033  *      When calling this method, interrupts MUST be enabled.  This is because
3034  *      the BH enable code must have IRQs enabled so that it will not deadlock.
3035  *          --BLG
3036  */
3037 static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv)
3038 {
3039         struct net_device *dev = skb->dev;
3040         struct netdev_queue *txq;
3041         struct Qdisc *q;
3042         int rc = -ENOMEM;
3043 
3044         skb_reset_mac_header(skb);
3045 
3046         if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
3047                 __skb_tstamp_tx(skb, NULL, skb->sk, SCM_TSTAMP_SCHED);
3048 
3049         /* Disable soft irqs for various locks below. Also
3050          * stops preemption for RCU.
3051          */
3052         rcu_read_lock_bh();
3053 
3054         skb_update_prio(skb);
3055 
3056         /* If device/qdisc don't need skb->dst, release it right now while
3057          * its hot in this cpu cache.
3058          */
3059         if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
3060                 skb_dst_drop(skb);
3061         else
3062                 skb_dst_force(skb);
3063 
3064         txq = netdev_pick_tx(dev, skb, accel_priv);
3065         q = rcu_dereference_bh(txq->qdisc);
3066 
3067 #ifdef CONFIG_NET_CLS_ACT
3068         skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS);
3069 #endif
3070         trace_net_dev_queue(skb);
3071         if (q->enqueue) {
3072                 rc = __dev_xmit_skb(skb, q, dev, txq);
3073                 goto out;
3074         }
3075 
3076         /* The device has no queue. Common case for software devices:
3077            loopback, all the sorts of tunnels...
3078 
3079            Really, it is unlikely that netif_tx_lock protection is necessary
3080            here.  (f.e. loopback and IP tunnels are clean ignoring statistics
3081            counters.)
3082            However, it is possible, that they rely on protection
3083            made by us here.
3084 
3085            Check this and shot the lock. It is not prone from deadlocks.
3086            Either shot noqueue qdisc, it is even simpler 8)
3087          */
3088         if (dev->flags & IFF_UP) {
3089                 int cpu = smp_processor_id(); /* ok because BHs are off */
3090 
3091                 if (txq->xmit_lock_owner != cpu) {
3092 
3093                         if (__this_cpu_read(xmit_recursion) > RECURSION_LIMIT)
3094                                 goto recursion_alert;
3095 
3096                         skb = validate_xmit_skb(skb, dev);
3097                         if (!skb)
3098                                 goto drop;
3099 
3100                         HARD_TX_LOCK(dev, txq, cpu);
3101 
3102                         if (!netif_xmit_stopped(txq)) {
3103                                 __this_cpu_inc(xmit_recursion);
3104                                 skb = dev_hard_start_xmit(skb, dev, txq, &rc);
3105                                 __this_cpu_dec(xmit_recursion);
3106                                 if (dev_xmit_complete(rc)) {
3107                                         HARD_TX_UNLOCK(dev, txq);
3108                                         goto out;
3109                                 }
3110                         }
3111                         HARD_TX_UNLOCK(dev, txq);
3112                         net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
3113                                              dev->name);
3114                 } else {
3115                         /* Recursion is detected! It is possible,
3116                          * unfortunately
3117                          */
3118 recursion_alert:
3119                         net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
3120                                              dev->name);
3121                 }
3122         }
3123 
3124         rc = -ENETDOWN;
3125 drop:
3126         rcu_read_unlock_bh();
3127 
3128         atomic_long_inc(&dev->tx_dropped);
3129         kfree_skb_list(skb);
3130         return rc;
3131 out:
3132         rcu_read_unlock_bh();
3133         return rc;
3134 }
3135 
3136 int dev_queue_xmit_sk(struct sock *sk, struct sk_buff *skb)
3137 {
3138         return __dev_queue_xmit(skb, NULL);
3139 }
3140 EXPORT_SYMBOL(dev_queue_xmit_sk);
3141 
3142 int dev_queue_xmit_accel(struct sk_buff *skb, void *accel_priv)
3143 {
3144         return __dev_queue_xmit(skb, accel_priv);
3145 }
3146 EXPORT_SYMBOL(dev_queue_xmit_accel);
3147 
3148 
3149 /*=======================================================================
3150                         Receiver routines
3151   =======================================================================*/
3152 
3153 int netdev_max_backlog __read_mostly = 1000;
3154 EXPORT_SYMBOL(netdev_max_backlog);
3155 
3156 int netdev_tstamp_prequeue __read_mostly = 1;
3157 int netdev_budget __read_mostly = 300;
3158 int weight_p __read_mostly = 64;            /* old backlog weight */
3159 
3160 /* Called with irq disabled */
3161 static inline void ____napi_schedule(struct softnet_data *sd,
3162                                      struct napi_struct *napi)
3163 {
3164         list_add_tail(&napi->poll_list, &sd->poll_list);
3165         __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3166 }
3167 
3168 #ifdef CONFIG_RPS
3169 
3170 /* One global table that all flow-based protocols share. */
3171 struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
3172 EXPORT_SYMBOL(rps_sock_flow_table);
3173 u32 rps_cpu_mask __read_mostly;
3174 EXPORT_SYMBOL(rps_cpu_mask);
3175 
3176 struct static_key rps_needed __read_mostly;
3177 
3178 static struct rps_dev_flow *
3179 set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3180             struct rps_dev_flow *rflow, u16 next_cpu)
3181 {
3182         if (next_cpu < nr_cpu_ids) {
3183 #ifdef CONFIG_RFS_ACCEL
3184                 struct netdev_rx_queue *rxqueue;
3185                 struct rps_dev_flow_table *flow_table;
3186                 struct rps_dev_flow *old_rflow;
3187                 u32 flow_id;
3188                 u16 rxq_index;
3189                 int rc;
3190 
3191                 /* Should we steer this flow to a different hardware queue? */
3192                 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
3193                     !(dev->features & NETIF_F_NTUPLE))
3194                         goto out;
3195                 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
3196                 if (rxq_index == skb_get_rx_queue(skb))
3197                         goto out;
3198 
3199                 rxqueue = dev->_rx + rxq_index;
3200                 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3201                 if (!flow_table)
3202                         goto out;
3203                 flow_id = skb_get_hash(skb) & flow_table->mask;
3204                 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
3205                                                         rxq_index, flow_id);
3206                 if (rc < 0)
3207                         goto out;
3208                 old_rflow = rflow;
3209                 rflow = &flow_table->flows[flow_id];
3210                 rflow->filter = rc;
3211                 if (old_rflow->filter == rflow->filter)
3212                         old_rflow->filter = RPS_NO_FILTER;
3213         out:
3214 #endif
3215                 rflow->last_qtail =
3216                         per_cpu(softnet_data, next_cpu).input_queue_head;
3217         }
3218 
3219         rflow->cpu = next_cpu;
3220         return rflow;
3221 }
3222 
3223 /*
3224  * get_rps_cpu is called from netif_receive_skb and returns the target
3225  * CPU from the RPS map of the receiving queue for a given skb.
3226  * rcu_read_lock must be held on entry.
3227  */
3228 static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3229                        struct rps_dev_flow **rflowp)
3230 {
3231         const struct rps_sock_flow_table *sock_flow_table;
3232         struct netdev_rx_queue *rxqueue = dev->_rx;
3233         struct rps_dev_flow_table *flow_table;
3234         struct rps_map *map;
3235         int cpu = -1;
3236         u32 tcpu;
3237         u32 hash;
3238 
3239         if (skb_rx_queue_recorded(skb)) {
3240                 u16 index = skb_get_rx_queue(skb);
3241 
3242                 if (unlikely(index >= dev->real_num_rx_queues)) {
3243                         WARN_ONCE(dev->real_num_rx_queues > 1,
3244                                   "%s received packet on queue %u, but number "
3245                                   "of RX queues is %u\n",
3246                                   dev->name, index, dev->real_num_rx_queues);
3247                         goto done;
3248                 }
3249                 rxqueue += index;
3250         }
3251 
3252         /* Avoid computing hash if RFS/RPS is not active for this rxqueue */
3253 
3254         flow_table = rcu_dereference(rxqueue->rps_flow_table);
3255         map = rcu_dereference(rxqueue->rps_map);
3256         if (!flow_table && !map)
3257                 goto done;
3258 
3259         skb_reset_network_header(skb);
3260         hash = skb_get_hash(skb);
3261         if (!hash)
3262                 goto done;
3263 
3264         sock_flow_table = rcu_dereference(rps_sock_flow_table);
3265         if (flow_table && sock_flow_table) {
3266                 struct rps_dev_flow *rflow;
3267                 u32 next_cpu;
3268                 u32 ident;
3269 
3270                 /* First check into global flow table if there is a match */
3271                 ident = sock_flow_table->ents[hash & sock_flow_table->mask];
3272                 if ((ident ^ hash) & ~rps_cpu_mask)
3273                         goto try_rps;
3274 
3275                 next_cpu = ident & rps_cpu_mask;
3276 
3277                 /* OK, now we know there is a match,
3278                  * we can look at the local (per receive queue) flow table
3279                  */
3280                 rflow = &flow_table->flows[hash & flow_table->mask];
3281                 tcpu = rflow->cpu;
3282 
3283                 /*
3284                  * If the desired CPU (where last recvmsg was done) is
3285                  * different from current CPU (one in the rx-queue flow
3286                  * table entry), switch if one of the following holds:
3287                  *   - Current CPU is unset (>= nr_cpu_ids).
3288                  *   - Current CPU is offline.
3289                  *   - The current CPU's queue tail has advanced beyond the
3290                  *     last packet that was enqueued using this table entry.
3291                  *     This guarantees that all previous packets for the flow
3292                  *     have been dequeued, thus preserving in order delivery.
3293                  */
3294                 if (unlikely(tcpu != next_cpu) &&
3295                     (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
3296                      ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
3297                       rflow->last_qtail)) >= 0)) {
3298                         tcpu = next_cpu;
3299                         rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
3300                 }
3301 
3302                 if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
3303                         *rflowp = rflow;
3304                         cpu = tcpu;
3305                         goto done;
3306                 }
3307         }
3308 
3309 try_rps:
3310 
3311         if (map) {
3312                 tcpu = map->cpus[reciprocal_scale(hash, map->len)];
3313                 if (cpu_online(tcpu)) {
3314                         cpu = tcpu;
3315                         goto done;
3316                 }
3317         }
3318 
3319 done:
3320         return cpu;
3321 }
3322 
3323 #ifdef CONFIG_RFS_ACCEL
3324 
3325 /**
3326  * rps_may_expire_flow - check whether an RFS hardware filter may be removed
3327  * @dev: Device on which the filter was set
3328  * @rxq_index: RX queue index
3329  * @flow_id: Flow ID passed to ndo_rx_flow_steer()
3330  * @filter_id: Filter ID returned by ndo_rx_flow_steer()
3331  *
3332  * Drivers that implement ndo_rx_flow_steer() should periodically call
3333  * this function for each installed filter and remove the filters for
3334  * which it returns %true.
3335  */
3336 bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
3337                          u32 flow_id, u16 filter_id)
3338 {
3339         struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
3340         struct rps_dev_flow_table *flow_table;
3341         struct rps_dev_flow *rflow;
3342         bool expire = true;
3343         unsigned int cpu;
3344 
3345         rcu_read_lock();
3346         flow_table = rcu_dereference(rxqueue->rps_flow_table);
3347         if (flow_table && flow_id <= flow_table->mask) {
3348                 rflow = &flow_table->flows[flow_id];
3349                 cpu = ACCESS_ONCE(rflow->cpu);
3350                 if (rflow->filter == filter_id && cpu < nr_cpu_ids &&
3351                     ((int)(per_cpu(softnet_data, cpu).input_queue_head -
3352                            rflow->last_qtail) <
3353                      (int)(10 * flow_table->mask)))
3354                         expire = false;
3355         }
3356         rcu_read_unlock();
3357         return expire;
3358 }
3359 EXPORT_SYMBOL(rps_may_expire_flow);
3360 
3361 #endif /* CONFIG_RFS_ACCEL */
3362 
3363 /* Called from hardirq (IPI) context */
3364 static void rps_trigger_softirq(void *data)
3365 {
3366         struct softnet_data *sd = data;
3367 
3368         ____napi_schedule(sd, &sd->backlog);
3369         sd->received_rps++;
3370 }
3371 
3372 #endif /* CONFIG_RPS */
3373 
3374 /*
3375  * Check if this softnet_data structure is another cpu one
3376  * If yes, queue it to our IPI list and return 1
3377  * If no, return 0
3378  */
3379 static int rps_ipi_queued(struct softnet_data *sd)
3380 {
3381 #ifdef CONFIG_RPS
3382         struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
3383 
3384         if (sd != mysd) {
3385                 sd->rps_ipi_next = mysd->rps_ipi_list;
3386                 mysd->rps_ipi_list = sd;
3387 
3388                 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3389                 return 1;
3390         }
3391 #endif /* CONFIG_RPS */
3392         return 0;
3393 }
3394 
3395 #ifdef CONFIG_NET_FLOW_LIMIT
3396 int netdev_flow_limit_table_len __read_mostly = (1 << 12);
3397 #endif
3398 
3399 static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
3400 {
3401 #ifdef CONFIG_NET_FLOW_LIMIT
3402         struct sd_flow_limit *fl;
3403         struct softnet_data *sd;
3404         unsigned int old_flow, new_flow;
3405 
3406         if (qlen < (netdev_max_backlog >> 1))
3407                 return false;
3408 
3409         sd = this_cpu_ptr(&softnet_data);
3410 
3411         rcu_read_lock();
3412         fl = rcu_dereference(sd->flow_limit);
3413         if (fl) {
3414                 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
3415                 old_flow = fl->history[fl->history_head];
3416                 fl->history[fl->history_head] = new_flow;
3417 
3418                 fl->history_head++;
3419                 fl->history_head &= FLOW_LIMIT_HISTORY - 1;
3420 
3421                 if (likely(fl->buckets[old_flow]))
3422                         fl->buckets[old_flow]--;
3423 
3424                 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
3425                         fl->count++;
3426                         rcu_read_unlock();
3427                         return true;
3428                 }
3429         }
3430         rcu_read_unlock();
3431 #endif
3432         return false;
3433 }
3434 
3435 /*
3436  * enqueue_to_backlog is called to queue an skb to a per CPU backlog
3437  * queue (may be a remote CPU queue).
3438  */
3439 static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
3440                               unsigned int *qtail)
3441 {
3442         struct softnet_data *sd;
3443         unsigned long flags;
3444         unsigned int qlen;
3445 
3446         sd = &per_cpu(softnet_data, cpu);
3447 
3448         local_irq_save(flags);
3449 
3450         rps_lock(sd);
3451         if (!netif_running(skb->dev))
3452                 goto drop;
3453         qlen = skb_queue_len(&sd->input_pkt_queue);
3454         if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) {
3455                 if (qlen) {
3456 enqueue:
3457                         __skb_queue_tail(&sd->input_pkt_queue, skb);
3458                         input_queue_tail_incr_save(sd, qtail);
3459                         rps_unlock(sd);
3460                         local_irq_restore(flags);
3461                         return NET_RX_SUCCESS;
3462                 }
3463 
3464                 /* Schedule NAPI for backlog device
3465                  * We can use non atomic operation since we own the queue lock
3466                  */
3467                 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
3468                         if (!rps_ipi_queued(sd))
3469                                 ____napi_schedule(sd, &sd->backlog);
3470                 }
3471                 goto enqueue;
3472         }
3473 
3474 drop:
3475         sd->dropped++;
3476         rps_unlock(sd);
3477 
3478         local_irq_restore(flags);
3479 
3480         atomic_long_inc(&skb->dev->rx_dropped);
3481         kfree_skb(skb);
3482         return NET_RX_DROP;
3483 }
3484 
3485 static int netif_rx_internal(struct sk_buff *skb)
3486 {
3487         int ret;
3488 
3489         net_timestamp_check(netdev_tstamp_prequeue, skb);
3490 
3491         trace_netif_rx(skb);
3492 #ifdef CONFIG_RPS
3493         if (static_key_false(&rps_needed)) {
3494                 struct rps_dev_flow voidflow, *rflow = &voidflow;
3495                 int cpu;
3496 
3497                 preempt_disable();
3498                 rcu_read_lock();
3499 
3500                 cpu = get_rps_cpu(skb->dev, skb, &rflow);
3501                 if (cpu < 0)
3502                         cpu = smp_processor_id();
3503 
3504                 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3505 
3506                 rcu_read_unlock();
3507                 preempt_enable();
3508         } else
3509 #endif
3510         {
3511                 unsigned int qtail;
3512                 ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
3513                 put_cpu();
3514         }
3515         return ret;
3516 }
3517 
3518 /**
3519  *      netif_rx        -       post buffer to the network code
3520  *      @skb: buffer to post
3521  *
3522  *      This function receives a packet from a device driver and queues it for
3523  *      the upper (protocol) levels to process.  It always succeeds. The buffer
3524  *      may be dropped during processing for congestion control or by the
3525  *      protocol layers.
3526  *
3527  *      return values:
3528  *      NET_RX_SUCCESS  (no congestion)
3529  *      NET_RX_DROP     (packet was dropped)
3530  *
3531  */
3532 
3533 int netif_rx(struct sk_buff *skb)
3534 {
3535         trace_netif_rx_entry(skb);
3536 
3537         return netif_rx_internal(skb);
3538 }
3539 EXPORT_SYMBOL(netif_rx);
3540 
3541 int netif_rx_ni(struct sk_buff *skb)
3542 {
3543         int err;
3544 
3545         trace_netif_rx_ni_entry(skb);
3546 
3547         preempt_disable();
3548         err = netif_rx_internal(skb);
3549         if (local_softirq_pending())
3550                 do_softirq();
3551         preempt_enable();
3552 
3553         return err;
3554 }
3555 EXPORT_SYMBOL(netif_rx_ni);
3556 
3557 static void net_tx_action(struct softirq_action *h)
3558 {
3559         struct softnet_data *sd = this_cpu_ptr(&softnet_data);
3560 
3561         if (sd->completion_queue) {
3562                 struct sk_buff *clist;
3563 
3564                 local_irq_disable();
3565                 clist = sd->completion_queue;
3566                 sd->completion_queue = NULL;
3567                 local_irq_enable();
3568 
3569                 while (clist) {
3570                         struct sk_buff *skb = clist;
3571                         clist = clist->next;
3572 
3573                         WARN_ON(atomic_read(&skb->users));
3574                         if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED))
3575                                 trace_consume_skb(skb);
3576                         else
3577                                 trace_kfree_skb(skb, net_tx_action);
3578                         __kfree_skb(skb);
3579                 }
3580         }
3581 
3582         if (sd->output_queue) {
3583                 struct Qdisc *head;
3584 
3585                 local_irq_disable();
3586                 head = sd->output_queue;
3587                 sd->output_queue = NULL;
3588                 sd->output_queue_tailp = &sd->output_queue;
3589                 local_irq_enable();
3590 
3591                 while (head) {
3592                         struct Qdisc *q = head;
3593                         spinlock_t *root_lock;
3594 
3595                         head = head->next_sched;
3596 
3597                         root_lock = qdisc_lock(q);
3598                         if (spin_trylock(root_lock)) {
3599                                 smp_mb__before_atomic();
3600                                 clear_bit(__QDISC_STATE_SCHED,
3601                                           &q->state);
3602                                 qdisc_run(q);
3603                                 spin_unlock(root_lock);
3604                         } else {
3605                                 if (!test_bit(__QDISC_STATE_DEACTIVATED,
3606                                               &q->state)) {
3607                                         __netif_reschedule(q);
3608                                 } else {
3609                                         smp_mb__before_atomic();
3610                                         clear_bit(__QDISC_STATE_SCHED,
3611                                                   &q->state);
3612                                 }
3613                         }
3614                 }
3615         }
3616 }
3617 
3618 #if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \
3619     (defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE))
3620 /* This hook is defined here for ATM LANE */
3621 int (*br_fdb_test_addr_hook)(struct net_device *dev,
3622                              unsigned char *addr) __read_mostly;
3623 EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
3624 #endif
3625 
3626 static inline struct sk_buff *handle_ing(struct sk_buff *skb,
3627                                          struct packet_type **pt_prev,
3628                                          int *ret, struct net_device *orig_dev)
3629 {
3630 #ifdef CONFIG_NET_CLS_ACT
3631         struct tcf_proto *cl = rcu_dereference_bh(skb->dev->ingress_cl_list);
3632         struct tcf_result cl_res;
3633 
3634         /* If there's at least one ingress present somewhere (so
3635          * we get here via enabled static key), remaining devices
3636          * that are not configured with an ingress qdisc will bail
3637          * out here.
3638          */
3639         if (!cl)
3640                 return skb;
3641         if (*pt_prev) {
3642                 *ret = deliver_skb(skb, *pt_prev, orig_dev);
3643                 *pt_prev = NULL;
3644         }
3645 
3646         qdisc_skb_cb(skb)->pkt_len = skb->len;
3647         skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS);
3648         qdisc_bstats_update_cpu(cl->q, skb);
3649 
3650         switch (tc_classify(skb, cl, &cl_res)) {
3651         case TC_ACT_OK:
3652         case TC_ACT_RECLASSIFY:
3653                 skb->tc_index = TC_H_MIN(cl_res.classid);
3654                 break;
3655         case TC_ACT_SHOT:
3656                 qdisc_qstats_drop_cpu(cl->q);
3657         case TC_ACT_STOLEN:
3658         case TC_ACT_QUEUED:
3659                 kfree_skb(skb);
3660                 return NULL;
3661         default:
3662                 break;
3663         }
3664 #endif /* CONFIG_NET_CLS_ACT */
3665         return skb;
3666 }
3667 
3668 /**
3669  *      netdev_rx_handler_register - register receive handler
3670  *      @dev: device to register a handler for
3671  *      @rx_handler: receive handler to register
3672  *      @rx_handler_data: data pointer that is used by rx handler
3673  *
3674  *      Register a receive handler for a device. This handler will then be
3675  *      called from __netif_receive_skb. A negative errno code is returned
3676  *      on a failure.
3677  *
3678  *      The caller must hold the rtnl_mutex.
3679  *
3680  *      For a general description of rx_handler, see enum rx_handler_result.
3681  */
3682 int netdev_rx_handler_register(struct net_device *dev,
3683                                rx_handler_func_t *rx_handler,
3684                                void *rx_handler_data)
3685 {
3686         ASSERT_RTNL();
3687 
3688         if (dev->rx_handler)
3689                 return -EBUSY;
3690 
3691         /* Note: rx_handler_data must be set before rx_handler */
3692         rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
3693         rcu_assign_pointer(dev->rx_handler, rx_handler);
3694 
3695         return 0;
3696 }
3697 EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
3698 
3699 /**
3700  *      netdev_rx_handler_unregister - unregister receive handler
3701  *      @dev: device to unregister a handler from
3702  *
3703  *      Unregister a receive handler from a device.
3704  *
3705  *      The caller must hold the rtnl_mutex.
3706  */
3707 void netdev_rx_handler_unregister(struct net_device *dev)
3708 {
3709 
3710         ASSERT_RTNL();
3711         RCU_INIT_POINTER(dev->rx_handler, NULL);
3712         /* a reader seeing a non NULL rx_handler in a rcu_read_lock()
3713          * section has a guarantee to see a non NULL rx_handler_data
3714          * as well.
3715          */
3716         synchronize_net();
3717         RCU_INIT_POINTER(dev->rx_handler_data, NULL);
3718 }
3719 EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
3720 
3721 /*
3722  * Limit the use of PFMEMALLOC reserves to those protocols that implement
3723  * the special handling of PFMEMALLOC skbs.
3724  */
3725 static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
3726 {
3727         switch (skb->protocol) {
3728         case htons(ETH_P_ARP):
3729         case htons(ETH_P_IP):
3730         case htons(ETH_P_IPV6):
3731         case htons(ETH_P_8021Q):
3732         case htons(ETH_P_8021AD):
3733                 return true;
3734         default:
3735                 return false;
3736         }
3737 }
3738 
3739 static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev,
3740                              int *ret, struct net_device *orig_dev)
3741 {
3742 #ifdef CONFIG_NETFILTER_INGRESS
3743         if (nf_hook_ingress_active(skb)) {
3744                 if (*pt_prev) {
3745                         *ret = deliver_skb(skb, *pt_prev, orig_dev);
3746                         *pt_prev = NULL;
3747                 }
3748 
3749                 return nf_hook_ingress(skb);
3750         }
3751 #endif /* CONFIG_NETFILTER_INGRESS */
3752         return 0;
3753 }
3754 
3755 static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
3756 {
3757         struct packet_type *ptype, *pt_prev;
3758         rx_handler_func_t *rx_handler;
3759         struct net_device *orig_dev;
3760         bool deliver_exact = false;
3761         int ret = NET_RX_DROP;
3762         __be16 type;
3763 
3764         net_timestamp_check(!netdev_tstamp_prequeue, skb);
3765 
3766         trace_netif_receive_skb(skb);
3767 
3768         orig_dev = skb->dev;
3769 
3770         skb_reset_network_header(skb);
3771         if (!skb_transport_header_was_set(skb))
3772                 skb_reset_transport_header(skb);
3773         skb_reset_mac_len(skb);
3774 
3775         pt_prev = NULL;
3776 
3777 another_round:
3778         skb->skb_iif = skb->dev->ifindex;
3779 
3780         __this_cpu_inc(softnet_data.processed);
3781 
3782         if (skb->protocol == cpu_to_be16(ETH_P_8021Q) ||
3783             skb->protocol == cpu_to_be16(ETH_P_8021AD)) {
3784                 skb = skb_vlan_untag(skb);
3785                 if (unlikely(!skb))
3786                         goto out;
3787         }
3788 
3789 #ifdef CONFIG_NET_CLS_ACT
3790         if (skb->tc_verd & TC_NCLS) {
3791                 skb->tc_verd = CLR_TC_NCLS(skb->tc_verd);
3792                 goto ncls;
3793         }
3794 #endif
3795 
3796         if (pfmemalloc)
3797                 goto skip_taps;
3798 
3799         list_for_each_entry_rcu(ptype, &ptype_all, list) {
3800                 if (pt_prev)
3801                         ret = deliver_skb(skb, pt_prev, orig_dev);
3802                 pt_prev = ptype;
3803         }
3804 
3805         list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
3806                 if (pt_prev)
3807                         ret = deliver_skb(skb, pt_prev, orig_dev);
3808                 pt_prev = ptype;
3809         }
3810 
3811 skip_taps:
3812 #ifdef CONFIG_NET_INGRESS
3813         if (static_key_false(&ingress_needed)) {
3814                 skb = handle_ing(skb, &pt_prev, &ret, orig_dev);
3815                 if (!skb)
3816                         goto out;
3817 
3818                 if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0)
3819                         goto out;
3820         }
3821 #endif
3822 #ifdef CONFIG_NET_CLS_ACT
3823         skb->tc_verd = 0;
3824 ncls:
3825 #endif
3826         if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
3827                 goto drop;
3828 
3829         if (skb_vlan_tag_present(skb)) {
3830                 if (pt_prev) {
3831                         ret = deliver_skb(skb, pt_prev, orig_dev);
3832                         pt_prev = NULL;
3833                 }
3834                 if (vlan_do_receive(&skb))
3835                         goto another_round;
3836                 else if (unlikely(!skb))
3837                         goto out;
3838         }
3839 
3840         rx_handler = rcu_dereference(skb->dev->rx_handler);
3841         if (rx_handler) {
3842                 if (pt_prev) {
3843                         ret = deliver_skb(skb, pt_prev, orig_dev);
3844                         pt_prev = NULL;
3845                 }
3846                 switch (rx_handler(&skb)) {
3847                 case RX_HANDLER_CONSUMED:
3848                         ret = NET_RX_SUCCESS;
3849                         goto out;
3850                 case RX_HANDLER_ANOTHER:
3851                         goto another_round;
3852                 case RX_HANDLER_EXACT:
3853                         deliver_exact = true;
3854                 case RX_HANDLER_PASS:
3855                         break;
3856                 default:
3857                         BUG();
3858                 }
3859         }
3860 
3861         if (unlikely(skb_vlan_tag_present(skb))) {
3862                 if (skb_vlan_tag_get_id(skb))
3863                         skb->pkt_type = PACKET_OTHERHOST;
3864                 /* Note: we might in the future use prio bits
3865                  * and set skb->priority like in vlan_do_receive()
3866                  * For the time being, just ignore Priority Code Point
3867                  */
3868                 skb->vlan_tci = 0;
3869         }
3870 
3871         type = skb->protocol;
3872 
3873         /* deliver only exact match when indicated */
3874         if (likely(!deliver_exact)) {
3875                 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
3876                                        &ptype_base[ntohs(type) &
3877                                                    PTYPE_HASH_MASK]);
3878         }
3879 
3880         deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
3881                                &orig_dev->ptype_specific);
3882 
3883         if (unlikely(skb->dev != orig_dev)) {
3884                 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
3885                                        &skb->dev->ptype_specific);
3886         }
3887 
3888         if (pt_prev) {
3889                 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
3890                         goto drop;
3891                 else
3892                         ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
3893         } else {
3894 drop:
3895                 atomic_long_inc(&skb->dev->rx_dropped);
3896                 kfree_skb(skb);
3897                 /* Jamal, now you will not able to escape explaining
3898                  * me how you were going to use this. :-)
3899                  */
3900                 ret = NET_RX_DROP;
3901         }
3902 
3903 out:
3904         return ret;
3905 }
3906 
3907 static int __netif_receive_skb(struct sk_buff *skb)
3908 {
3909         int ret;
3910 
3911         if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
3912                 unsigned long pflags = current->flags;
3913 
3914                 /*
3915                  * PFMEMALLOC skbs are special, they should
3916                  * - be delivered to SOCK_MEMALLOC sockets only
3917                  * - stay away from userspace
3918                  * - have bounded memory usage
3919                  *
3920                  * Use PF_MEMALLOC as this saves us from propagating the allocation
3921                  * context down to all allocation sites.
3922                  */
3923                 current->flags |= PF_MEMALLOC;
3924                 ret = __netif_receive_skb_core(skb, true);
3925                 tsk_restore_flags(current, pflags, PF_MEMALLOC);
3926         } else
3927                 ret = __netif_receive_skb_core(skb, false);
3928 
3929         return ret;
3930 }
3931 
3932 static int netif_receive_skb_internal(struct sk_buff *skb)
3933 {
3934         int ret;
3935 
3936         net_timestamp_check(netdev_tstamp_prequeue, skb);
3937 
3938         if (skb_defer_rx_timestamp(skb))
3939                 return NET_RX_SUCCESS;
3940 
3941         rcu_read_lock();
3942 
3943 #ifdef CONFIG_RPS
3944         if (static_key_false(&rps_needed)) {
3945                 struct rps_dev_flow voidflow, *rflow = &voidflow;
3946                 int cpu = get_rps_cpu(skb->dev, skb, &rflow);
3947 
3948                 if (cpu >= 0) {
3949                         ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3950                         rcu_read_unlock();
3951                         return ret;
3952                 }
3953         }
3954 #endif
3955         ret = __netif_receive_skb(skb);
3956         rcu_read_unlock();
3957         return ret;
3958 }
3959 
3960 /**
3961  *      netif_receive_skb - process receive buffer from network
3962  *      @skb: buffer to process
3963  *
3964  *      netif_receive_skb() is the main receive data processing function.
3965  *      It always succeeds. The buffer may be dropped during processing
3966  *      for congestion control or by the protocol layers.
3967  *
3968  *      This function may only be called from softirq context and interrupts
3969  *      should be enabled.
3970  *
3971  *      Return values (usually ignored):
3972  *      NET_RX_SUCCESS: no congestion
3973  *      NET_RX_DROP: packet was dropped
3974  */
3975 int netif_receive_skb_sk(struct sock *sk, struct sk_buff *skb)
3976 {
3977         trace_netif_receive_skb_entry(skb);
3978 
3979         return netif_receive_skb_internal(skb);
3980 }
3981 EXPORT_SYMBOL(netif_receive_skb_sk);
3982 
3983 /* Network device is going away, flush any packets still pending
3984  * Called with irqs disabled.
3985  */
3986 static void flush_backlog(void *arg)
3987 {
3988         struct net_device *dev = arg;
3989         struct softnet_data *sd = this_cpu_ptr(&softnet_data);
3990         struct sk_buff *skb, *tmp;
3991 
3992         rps_lock(sd);
3993         skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
3994                 if (skb->dev == dev) {
3995                         __skb_unlink(skb, &sd->input_pkt_queue);
3996                         kfree_skb(skb);
3997                         input_queue_head_incr(sd);
3998                 }
3999         }
4000         rps_unlock(sd);
4001 
4002         skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
4003                 if (skb->dev == dev) {
4004                         __skb_unlink(skb, &sd->process_queue);
4005                         kfree_skb(skb);
4006                         input_queue_head_incr(sd);
4007                 }
4008         }
4009 }
4010 
4011 static int napi_gro_complete(struct sk_buff *skb)
4012 {
4013         struct packet_offload *ptype;
4014         __be16 type = skb->protocol;
4015         struct list_head *head = &offload_base;
4016         int err = -ENOENT;
4017 
4018         BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));
4019 
4020         if (NAPI_GRO_CB(skb)->count == 1) {
4021                 skb_shinfo(skb)->gso_size = 0;
4022                 goto out;
4023         }
4024 
4025         rcu_read_lock();
4026         list_for_each_entry_rcu(ptype, head, list) {
4027                 if (ptype->type != type || !ptype->callbacks.gro_complete)
4028                         continue;
4029 
4030                 err = ptype->callbacks.gro_complete(skb, 0);
4031                 break;
4032         }
4033         rcu_read_unlock();
4034 
4035         if (err) {
4036                 WARN_ON(&ptype->list == head);
4037                 kfree_skb(skb);
4038                 return NET_RX_SUCCESS;
4039         }
4040 
4041 out:
4042         return netif_receive_skb_internal(skb);
4043 }
4044 
4045 /* napi->gro_list contains packets ordered by age.
4046  * youngest packets at the head of it.
4047  * Complete skbs in reverse order to reduce latencies.
4048  */
4049 void napi_gro_flush(struct napi_struct *napi, bool flush_old)
4050 {
4051         struct sk_buff *skb, *prev = NULL;
4052 
4053         /* scan list and build reverse chain */
4054         for (skb = napi->gro_list; skb != NULL; skb = skb->next) {
4055                 skb->prev = prev;
4056                 prev = skb;
4057         }
4058 
4059         for (skb = prev; skb; skb = prev) {
4060                 skb->next = NULL;
4061 
4062                 if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
4063                         return;
4064 
4065                 prev = skb->prev;
4066                 napi_gro_complete(skb);
4067                 napi->gro_count--;
4068         }
4069 
4070         napi->gro_list = NULL;
4071 }
4072 EXPORT_SYMBOL(napi_gro_flush);
4073 
4074 static void gro_list_prepare(struct napi_struct *napi, struct sk_buff *skb)
4075 {
4076         struct sk_buff *p;
4077         unsigned int maclen = skb->dev->hard_header_len;
4078         u32 hash = skb_get_hash_raw(skb);
4079 
4080         for (p = napi->gro_list; p; p = p->next) {
4081                 unsigned long diffs;
4082 
4083                 NAPI_GRO_CB(p)->flush = 0;
4084 
4085                 if (hash != skb_get_hash_raw(p)) {
4086                         NAPI_GRO_CB(p)->same_flow = 0;
4087                         continue;
4088                 }
4089 
4090                 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
4091                 diffs |= p->vlan_tci ^ skb->vlan_tci;
4092                 if (maclen == ETH_HLEN)
4093                         diffs |= compare_ether_header(skb_mac_header(p),
4094                                                       skb_mac_header(skb));
4095                 else if (!diffs)
4096                         diffs = memcmp(skb_mac_header(p),
4097                                        skb_mac_header(skb),
4098                                        maclen);
4099                 NAPI_GRO_CB(p)->same_flow = !diffs;
4100         }
4101 }
4102 
4103 static void skb_gro_reset_offset(struct sk_buff *skb)
4104 {
4105         const struct skb_shared_info *pinfo = skb_shinfo(skb);
4106         const skb_frag_t *frag0 = &pinfo->frags[0];
4107 
4108         NAPI_GRO_CB(skb)->data_offset = 0;
4109         NAPI_GRO_CB(skb)->frag0 = NULL;
4110         NAPI_GRO_CB(skb)->frag0_len = 0;
4111 
4112         if (skb_mac_header(skb) == skb_tail_pointer(skb) &&
4113             pinfo->nr_frags &&
4114             !PageHighMem(skb_frag_page(frag0))) {
4115                 NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
4116                 NAPI_GRO_CB(skb)->frag0_len = skb_frag_size(frag0);
4117         }
4118 }
4119 
4120 static void gro_pull_from_frag0(struct sk_buff *skb, int grow)
4121 {
4122         struct skb_shared_info *pinfo = skb_shinfo(skb);
4123 
4124         BUG_ON(skb->end - skb->tail < grow);
4125 
4126         memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
4127 
4128         skb->data_len -= grow;
4129         skb->tail += grow;
4130 
4131         pinfo->frags[0].page_offset += grow;
4132         skb_frag_size_sub(&pinfo->frags[0], grow);
4133 
4134         if (unlikely(!skb_frag_size(&pinfo->frags[0]))) {
4135                 skb_frag_unref(skb, 0);
4136                 memmove(pinfo->frags, pinfo->frags + 1,
4137                         --pinfo->nr_frags * sizeof(pinfo->frags[0]));
4138         }
4139 }
4140 
4141 static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
4142 {
4143         struct sk_buff **pp = NULL;
4144         struct packet_offload *ptype;
4145         __be16 type = skb->protocol;
4146         struct list_head *head = &offload_base;
4147         int same_flow;
4148         enum gro_result ret;
4149         int grow;
4150 
4151         if (!(skb->dev->features & NETIF_F_GRO))
4152                 goto normal;
4153 
4154         if (skb_is_gso(skb) || skb_has_frag_list(skb) || skb->csum_bad)
4155                 goto normal;
4156 
4157         gro_list_prepare(napi, skb);
4158 
4159         rcu_read_lock();
4160         list_for_each_entry_rcu(ptype, head, list) {
4161                 if (ptype->type != type || !ptype->callbacks.gro_receive)
4162                         continue;
4163 
4164                 skb_set_network_header(skb, skb_gro_offset(skb));
4165                 skb_reset_mac_len(skb);
4166                 NAPI_GRO_CB(skb)->same_flow = 0;
4167                 NAPI_GRO_CB(skb)->flush = 0;
4168                 NAPI_GRO_CB(skb)->free = 0;
4169                 NAPI_GRO_CB(skb)->udp_mark = 0;
4170                 NAPI_GRO_CB(skb)->gro_remcsum_start = 0;
4171 
4172                 /* Setup for GRO checksum validation */
4173                 switch (skb->ip_summed) {
4174                 case CHECKSUM_COMPLETE:
4175                         NAPI_GRO_CB(skb)->csum = skb->csum;
4176                         NAPI_GRO_CB(skb)->csum_valid = 1;
4177                         NAPI_GRO_CB(skb)->csum_cnt = 0;
4178                         break;
4179                 case CHECKSUM_UNNECESSARY:
4180                         NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1;
4181                         NAPI_GRO_CB(skb)->csum_valid = 0;
4182                         break;
4183                 default:
4184                         NAPI_GRO_CB(skb)->csum_cnt = 0;
4185                         NAPI_GRO_CB(skb)->csum_valid = 0;
4186                 }
4187 
4188                 pp = ptype->callbacks.gro_receive(&napi->gro_list, skb);
4189                 break;
4190         }
4191         rcu_read_unlock();
4192 
4193         if (&ptype->list == head)
4194                 goto normal;
4195 
4196         same_flow = NAPI_GRO_CB(skb)->same_flow;
4197         ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
4198 
4199         if (pp) {
4200                 struct sk_buff *nskb = *pp;
4201 
4202                 *pp = nskb->next;
4203                 nskb->next = NULL;
4204                 napi_gro_complete(nskb);
4205                 napi->gro_count--;
4206         }
4207 
4208         if (same_flow)
4209                 goto ok;
4210 
4211         if (NAPI_GRO_CB(skb)->flush)
4212                 goto normal;
4213 
4214         if (unlikely(napi->gro_count >= MAX_GRO_SKBS)) {
4215                 struct sk_buff *nskb = napi->gro_list;
4216 
4217                 /* locate the end of the list to select the 'oldest' flow */
4218                 while (nskb->next) {
4219                         pp = &nskb->next;
4220                         nskb = *pp;
4221                 }
4222                 *pp = NULL;
4223                 nskb->next = NULL;
4224                 napi_gro_complete(nskb);
4225         } else {
4226                 napi->gro_count++;
4227         }
4228         NAPI_GRO_CB(skb)->count = 1;
4229         NAPI_GRO_CB(skb)->age = jiffies;
4230         NAPI_GRO_CB(skb)->last = skb;
4231         skb_shinfo(skb)->gso_size = skb_gro_len(skb);
4232         skb->next = napi->gro_list;
4233         napi->gro_list = skb;
4234         ret = GRO_HELD;
4235 
4236 pull:
4237         grow = skb_gro_offset(skb) - skb_headlen(skb);
4238         if (grow > 0)
4239                 gro_pull_from_frag0(skb, grow);
4240 ok:
4241         return ret;
4242 
4243 normal:
4244         ret = GRO_NORMAL;
4245         goto pull;
4246 }
4247 
4248 struct packet_offload *gro_find_receive_by_type(__be16 type)
4249 {
4250         struct list_head *offload_head = &offload_base;
4251         struct packet_offload *ptype;
4252 
4253         list_for_each_entry_rcu(ptype, offload_head, list) {
4254                 if (ptype->type != type || !ptype->callbacks.gro_receive)
4255                         continue;
4256                 return ptype;
4257         }
4258         return NULL;
4259 }
4260 EXPORT_SYMBOL(gro_find_receive_by_type);
4261 
4262 struct packet_offload *gro_find_complete_by_type(__be16 type)
4263 {
4264         struct list_head *offload_head = &offload_base;
4265         struct packet_offload *ptype;
4266 
4267         list_for_each_entry_rcu(ptype, offload_head, list) {
4268                 if (ptype->type != type || !ptype->callbacks.gro_complete)
4269                         continue;
4270                 return ptype;
4271         }
4272         return NULL;
4273 }
4274 EXPORT_SYMBOL(gro_find_complete_by_type);
4275 
4276 static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
4277 {
4278         switch (ret) {
4279         case GRO_NORMAL:
4280                 if (netif_receive_skb_internal(skb))
4281                         ret = GRO_DROP;
4282                 break;
4283 
4284         case GRO_DROP:
4285                 kfree_skb(skb);
4286                 break;
4287 
4288         case GRO_MERGED_FREE:
4289                 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
4290                         kmem_cache_free(skbuff_head_cache, skb);
4291                 else
4292                         __kfree_skb(skb);
4293                 break;
4294 
4295         case GRO_HELD:
4296         case GRO_MERGED:
4297                 break;
4298         }
4299 
4300         return ret;
4301 }
4302 
4303 gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
4304 {
4305         trace_napi_gro_receive_entry(skb);
4306 
4307         skb_gro_reset_offset(skb);
4308 
4309         return napi_skb_finish(dev_gro_receive(napi, skb), skb);
4310 }
4311 EXPORT_SYMBOL(napi_gro_receive);
4312 
4313 static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
4314 {
4315         if (unlikely(skb->pfmemalloc)) {
4316                 consume_skb(skb);
4317                 return;
4318         }
4319         __skb_pull(skb, skb_headlen(skb));
4320         /* restore the reserve we had after netdev_alloc_skb_ip_align() */
4321         skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
4322         skb->vlan_tci = 0;
4323         skb->dev = napi->dev;
4324         skb->skb_iif = 0;
4325         skb->encapsulation = 0;
4326         skb_shinfo(skb)->gso_type = 0;
4327         skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
4328 
4329         napi->skb = skb;
4330 }
4331 
4332 struct sk_buff *napi_get_frags(struct napi_struct *napi)
4333 {
4334         struct sk_buff *skb = napi->skb;
4335 
4336         if (!skb) {
4337                 skb = napi_alloc_skb(napi, GRO_MAX_HEAD);
4338                 napi->skb = skb;
4339         }
4340         return skb;
4341 }
4342 EXPORT_SYMBOL(napi_get_frags);
4343 
4344 static gro_result_t napi_frags_finish(struct napi_struct *napi,
4345                                       struct sk_buff *skb,
4346                                       gro_result_t ret)
4347 {
4348         switch (ret) {
4349         case GRO_NORMAL:
4350         case GRO_HELD:
4351                 __skb_push(skb, ETH_HLEN);
4352                 skb->protocol = eth_type_trans(skb, skb->dev);
4353                 if (ret == GRO_NORMAL && netif_receive_skb_internal(skb))
4354                         ret = GRO_DROP;
4355                 break;
4356 
4357         case GRO_DROP:
4358         case GRO_MERGED_FREE:
4359                 napi_reuse_skb(napi, skb);
4360                 break;
4361 
4362         case GRO_MERGED:
4363                 break;
4364         }
4365 
4366         return ret;
4367 }
4368 
4369 /* Upper GRO stack assumes network header starts at gro_offset=0
4370  * Drivers could call both napi_gro_frags() and napi_gro_receive()
4371  * We copy ethernet header into skb->data to have a common layout.
4372  */
4373 static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
4374 {
4375         struct sk_buff *skb = napi->skb;
4376         const struct ethhdr *eth;
4377         unsigned int hlen = sizeof(*eth);
4378 
4379         napi->skb = NULL;
4380 
4381         skb_reset_mac_header(skb);
4382         skb_gro_reset_offset(skb);
4383 
4384         eth = skb_gro_header_fast(skb, 0);
4385         if (unlikely(skb_gro_header_hard(skb, hlen))) {
4386                 eth = skb_gro_header_slow(skb, hlen, 0);
4387                 if (unlikely(!eth)) {
4388                         napi_reuse_skb(napi, skb);
4389                         return NULL;
4390                 }
4391         } else {
4392                 gro_pull_from_frag0(skb, hlen);
4393                 NAPI_GRO_CB(skb)->frag0 += hlen;
4394                 NAPI_GRO_CB(skb)->frag0_len -= hlen;
4395         }
4396         __skb_pull(skb, hlen);
4397 
4398         /*
4399          * This works because the only protocols we care about don't require
4400          * special handling.
4401          * We'll fix it up properly in napi_frags_finish()
4402          */
4403         skb->protocol = eth->h_proto;
4404 
4405         return skb;
4406 }
4407 
4408 gro_result_t napi_gro_frags(struct napi_struct *napi)
4409 {
4410         struct sk_buff *skb = napi_frags_skb(napi);
4411 
4412         if (!skb)
4413                 return GRO_DROP;
4414 
4415         trace_napi_gro_frags_entry(skb);
4416 
4417         return napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));
4418 }
4419 EXPORT_SYMBOL(napi_gro_frags);
4420 
4421 /* Compute the checksum from gro_offset and return the folded value
4422  * after adding in any pseudo checksum.
4423  */
4424 __sum16 __skb_gro_checksum_complete(struct sk_buff *skb)
4425 {
4426         __wsum wsum;
4427         __sum16 sum;
4428 
4429         wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0);
4430 
4431         /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */
4432         sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum));
4433         if (likely(!sum)) {
4434                 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
4435                     !skb->csum_complete_sw)
4436                         netdev_rx_csum_fault(skb->dev);
4437         }
4438 
4439         NAPI_GRO_CB(skb)->csum = wsum;
4440         NAPI_GRO_CB(skb)->csum_valid = 1;
4441 
4442         return sum;
4443 }
4444 EXPORT_SYMBOL(__skb_gro_checksum_complete);
4445 
4446 /*
4447  * net_rps_action_and_irq_enable sends any pending IPI's for rps.
4448  * Note: called with local irq disabled, but exits with local irq enabled.
4449  */
4450 static void net_rps_action_and_irq_enable(struct softnet_data *sd)
4451 {
4452 #ifdef CONFIG_RPS
4453         struct softnet_data *remsd = sd->rps_ipi_list;
4454 
4455         if (remsd) {
4456                 sd->rps_ipi_list = NULL;
4457 
4458                 local_irq_enable();
4459 
4460                 /* Send pending IPI's to kick RPS processing on remote cpus. */
4461                 while (remsd) {
4462                         struct softnet_data *next = remsd->rps_ipi_next;
4463 
4464                         if (cpu_online(remsd->cpu))
4465                                 smp_call_function_single_async(remsd->cpu,
4466                                                            &remsd->csd);
4467                         remsd = next;
4468                 }
4469         } else
4470 #endif
4471                 local_irq_enable();
4472 }
4473 
4474 static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
4475 {
4476 #ifdef CONFIG_RPS
4477         return sd->rps_ipi_list != NULL;
4478 #else
4479         return false;
4480 #endif
4481 }
4482 
4483 static int process_backlog(struct napi_struct *napi, int quota)
4484 {
4485         int work = 0;
4486         struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
4487 
4488         /* Check if we have pending ipi, its better to send them now,
4489          * not waiting net_rx_action() end.
4490          */
4491         if (sd_has_rps_ipi_waiting(sd)) {
4492                 local_irq_disable();
4493                 net_rps_action_and_irq_enable(sd);
4494         }
4495 
4496         napi->weight = weight_p;
4497         local_irq_disable();
4498         while (1) {
4499                 struct sk_buff *skb;
4500 
4501                 while ((skb = __skb_dequeue(&sd->process_queue))) {
4502                         rcu_read_lock();
4503                         local_irq_enable();
4504                         __netif_receive_skb(skb);
4505                         rcu_read_unlock();
4506                         local_irq_disable();
4507                         input_queue_head_incr(sd);
4508                         if (++work >= quota) {
4509                                 local_irq_enable();
4510                                 return work;
4511                         }
4512                 }
4513 
4514                 rps_lock(sd);
4515                 if (skb_queue_empty(&sd->input_pkt_queue)) {
4516                         /*
4517                          * Inline a custom version of __napi_complete().
4518                          * only current cpu owns and manipulates this napi,
4519                          * and NAPI_STATE_SCHED is the only possible flag set
4520                          * on backlog.
4521                          * We can use a plain write instead of clear_bit(),
4522                          * and we dont need an smp_mb() memory barrier.
4523                          */
4524                         napi->state = 0;
4525                         rps_unlock(sd);
4526 
4527                         break;
4528                 }
4529 
4530                 skb_queue_splice_tail_init(&sd->input_pkt_queue,
4531                                            &sd->process_queue);
4532                 rps_unlock(sd);
4533         }
4534         local_irq_enable();
4535 
4536         return work;
4537 }
4538 
4539 /**
4540  * __napi_schedule - schedule for receive
4541  * @n: entry to schedule
4542  *
4543  * The entry's receive function will be scheduled to run.
4544  * Consider using __napi_schedule_irqoff() if hard irqs are masked.
4545  */
4546 void __napi_schedule(struct napi_struct *n)
4547 {
4548         unsigned long flags;
4549 
4550         local_irq_save(flags);
4551         ____napi_schedule(this_cpu_ptr(&softnet_data), n);
4552         local_irq_restore(flags);
4553 }
4554 EXPORT_SYMBOL(__napi_schedule);
4555 
4556 /**
4557  * __napi_schedule_irqoff - schedule for receive
4558  * @n: entry to schedule
4559  *
4560  * Variant of __napi_schedule() assuming hard irqs are masked
4561  */
4562 void __napi_schedule_irqoff(struct napi_struct *n)
4563 {
4564         ____napi_schedule(this_cpu_ptr(&softnet_data), n);
4565 }
4566 EXPORT_SYMBOL(__napi_schedule_irqoff);
4567 
4568 void __napi_complete(struct napi_struct *n)
4569 {
4570         BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
4571 
4572         list_del_init(&n->poll_list);
4573         smp_mb__before_atomic();
4574         clear_bit(NAPI_STATE_SCHED, &n->state);
4575 }
4576 EXPORT_SYMBOL(__napi_complete);
4577 
4578 void napi_complete_done(struct napi_struct *n, int work_done)
4579 {
4580         unsigned long flags;
4581 
4582         /*
4583          * don't let napi dequeue from the cpu poll list
4584          * just in case its running on a different cpu
4585          */
4586         if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state)))
4587                 return;
4588 
4589         if (n->gro_list) {
4590                 unsigned long timeout = 0;
4591 
4592                 if (work_done)
4593                         timeout = n->dev->gro_flush_timeout;
4594 
4595                 if (timeout)
4596                         hrtimer_start(&n->timer, ns_to_ktime(timeout),
4597                                       HRTIMER_MODE_REL_PINNED);
4598                 else
4599                         napi_gro_flush(n, false);
4600         }
4601         if (likely(list_empty(&n->poll_list))) {
4602                 WARN_ON_ONCE(!test_and_clear_bit(NAPI_STATE_SCHED, &n->state));
4603         } else {
4604                 /* If n->poll_list is not empty, we need to mask irqs */
4605                 local_irq_save(flags);
4606                 __napi_complete(n);
4607                 local_irq_restore(flags);
4608         }
4609 }
4610 EXPORT_SYMBOL(napi_complete_done);
4611 
4612 /* must be called under rcu_read_lock(), as we dont take a reference */
4613 struct napi_struct *napi_by_id(unsigned int napi_id)
4614 {
4615         unsigned int hash = napi_id % HASH_SIZE(napi_hash);
4616         struct napi_struct *napi;
4617 
4618         hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
4619                 if (napi->napi_id == napi_id)
4620                         return napi;
4621 
4622         return NULL;
4623 }
4624 EXPORT_SYMBOL_GPL(napi_by_id);
4625 
4626 void napi_hash_add(struct napi_struct *napi)
4627 {
4628         if (!test_and_set_bit(NAPI_STATE_HASHED, &napi->state)) {
4629 
4630                 spin_lock(&napi_hash_lock);
4631 
4632                 /* 0 is not a valid id, we also skip an id that is taken
4633                  * we expect both events to be extremely rare
4634                  */
4635                 napi->napi_id = 0;
4636                 while (!napi->napi_id) {
4637                         napi->napi_id = ++napi_gen_id;
4638                         if (napi_by_id(napi->napi_id))
4639                                 napi->napi_id = 0;
4640                 }
4641 
4642                 hlist_add_head_rcu(&napi->napi_hash_node,
4643                         &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
4644 
4645                 spin_unlock(&napi_hash_lock);
4646         }
4647 }
4648 EXPORT_SYMBOL_GPL(napi_hash_add);
4649 
4650 /* Warning : caller is responsible to make sure rcu grace period
4651  * is respected before freeing memory containing @napi
4652  */
4653 void napi_hash_del(struct napi_struct *napi)
4654 {
4655         spin_lock(&napi_hash_lock);
4656 
4657         if (test_and_clear_bit(NAPI_STATE_HASHED, &napi->state))
4658                 hlist_del_rcu(&napi->napi_hash_node);
4659 
4660         spin_unlock(&napi_hash_lock);
4661 }
4662 EXPORT_SYMBOL_GPL(napi_hash_del);
4663 
4664 static enum hrtimer_restart napi_watchdog(struct hrtimer *timer)
4665 {
4666         struct napi_struct *napi;
4667 
4668         napi = container_of(timer, struct napi_struct, timer);
4669         if (napi->gro_list)
4670                 napi_schedule(napi);
4671 
4672         return HRTIMER_NORESTART;
4673 }
4674 
4675 void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
4676                     int (*poll)(struct napi_struct *, int), int weight)
4677 {
4678         INIT_LIST_HEAD(&napi->poll_list);
4679         hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
4680         napi->timer.function = napi_watchdog;
4681         napi->gro_count = 0;
4682         napi->gro_list = NULL;
4683         napi->skb = NULL;
4684         napi->poll = poll;
4685         if (weight > NAPI_POLL_WEIGHT)
4686                 pr_err_once("netif_napi_add() called with weight %d on device %s\n",
4687                             weight, dev->name);
4688         napi->weight = weight;
4689         list_add(&napi->dev_list, &dev->napi_list);
4690         napi->dev = dev;
4691 #ifdef CONFIG_NETPOLL
4692         spin_lock_init(&napi->poll_lock);
4693         napi->poll_owner = -1;
4694 #endif
4695         set_bit(NAPI_STATE_SCHED, &napi->state);
4696 }
4697 EXPORT_SYMBOL(netif_napi_add);
4698 
4699 void napi_disable(struct napi_struct *n)
4700 {
4701         might_sleep();
4702         set_bit(NAPI_STATE_DISABLE, &n->state);
4703 
4704         while (test_and_set_bit(NAPI_STATE_SCHED, &n->state))
4705                 msleep(1);
4706 
4707         hrtimer_cancel(&n->timer);
4708 
4709         clear_bit(NAPI_STATE_DISABLE, &n->state);
4710 }
4711 EXPORT_SYMBOL(napi_disable);
4712 
4713 void netif_napi_del(struct napi_struct *napi)
4714 {
4715         list_del_init(&napi->dev_list);
4716         napi_free_frags(napi);
4717 
4718         kfree_skb_list(napi->gro_list);
4719         napi->gro_list = NULL;
4720         napi->gro_count = 0;
4721 }
4722 EXPORT_SYMBOL(netif_napi_del);
4723 
4724 static int napi_poll(struct napi_struct *n, struct list_head *repoll)
4725 {
4726         void *have;
4727         int work, weight;
4728 
4729         list_del_init(&n->poll_list);
4730 
4731         have = netpoll_poll_lock(n);
4732 
4733         weight = n->weight;
4734 
4735         /* This NAPI_STATE_SCHED test is for avoiding a race
4736          * with netpoll's poll_napi().  Only the entity which
4737          * obtains the lock and sees NAPI_STATE_SCHED set will
4738          * actually make the ->poll() call.  Therefore we avoid
4739          * accidentally calling ->poll() when NAPI is not scheduled.
4740          */
4741         work = 0;
4742         if (test_bit(NAPI_STATE_SCHED, &n->state)) {
4743                 work = n->poll(n, weight);
4744                 trace_napi_poll(n);
4745         }
4746 
4747         WARN_ON_ONCE(work > weight);
4748 
4749         if (likely(work < weight))
4750                 goto out_unlock;
4751 
4752         /* Drivers must not modify the NAPI state if they
4753          * consume the entire weight.  In such cases this code
4754          * still "owns" the NAPI instance and therefore can
4755          * move the instance around on the list at-will.
4756          */
4757         if (unlikely(napi_disable_pending(n))) {
4758                 napi_complete(n);
4759                 goto out_unlock;
4760         }
4761 
4762         if (n->gro_list) {
4763                 /* flush too old packets
4764                  * If HZ < 1000, flush all packets.
4765                  */
4766                 napi_gro_flush(n, HZ >= 1000);
4767         }
4768 
4769         /* Some drivers may have called napi_schedule
4770          * prior to exhausting their budget.
4771          */
4772         if (unlikely(!list_empty(&n->poll_list))) {
4773                 pr_warn_once("%s: Budget exhausted after napi rescheduled\n",
4774                              n->dev ? n->dev->name : "backlog");
4775                 goto out_unlock;
4776         }
4777 
4778         list_add_tail(&n->poll_list, repoll);
4779 
4780 out_unlock:
4781         netpoll_poll_unlock(have);
4782 
4783         return work;
4784 }
4785 
4786 static void net_rx_action(struct softirq_action *h)
4787 {
4788         struct softnet_data *sd = this_cpu_ptr(&softnet_data);
4789         unsigned long time_limit = jiffies + 2;
4790         int budget = netdev_budget;
4791         LIST_HEAD(list);
4792         LIST_HEAD(repoll);
4793 
4794         local_irq_disable();
4795         list_splice_init(&sd->poll_list, &list);
4796         local_irq_enable();
4797 
4798         for (;;) {
4799                 struct napi_struct *n;
4800 
4801                 if (list_empty(&list)) {
4802                         if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll))
4803                                 return;
4804                         break;
4805                 }
4806 
4807                 n = list_first_entry(&list, struct napi_struct, poll_list);
4808                 budget -= napi_poll(n, &repoll);
4809 
4810                 /* If softirq window is exhausted then punt.
4811                  * Allow this to run for 2 jiffies since which will allow
4812                  * an average latency of 1.5/HZ.
4813                  */
4814                 if (unlikely(budget <= 0 ||
4815                              time_after_eq(jiffies, time_limit))) {
4816                         sd->time_squeeze++;
4817                         break;
4818                 }
4819         }
4820 
4821         local_irq_disable();
4822 
4823         list_splice_tail_init(&sd->poll_list, &list);
4824         list_splice_tail(&repoll, &list);
4825         list_splice(&list, &sd->poll_list);
4826         if (!list_empty(&sd->poll_list))
4827                 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4828 
4829         net_rps_action_and_irq_enable(sd);
4830 }
4831 
4832 struct netdev_adjacent {
4833         struct net_device *dev;
4834 
4835         /* upper master flag, there can only be one master device per list */
4836         bool master;
4837 
4838         /* counter for the number of times this device was added to us */
4839         u16 ref_nr;
4840 
4841         /* private field for the users */
4842         void *private;
4843 
4844         struct list_head list;
4845         struct rcu_head rcu;
4846 };
4847 
4848 static struct netdev_adjacent *__netdev_find_adj(struct net_device *dev,
4849                                                  struct net_device *adj_dev,
4850                                                  struct list_head *adj_list)
4851 {
4852         struct netdev_adjacent *adj;
4853 
4854         list_for_each_entry(adj, adj_list, list) {
4855                 if (adj->dev == adj_dev)
4856                         return adj;
4857         }
4858         return NULL;
4859 }
4860 
4861 /**
4862  * netdev_has_upper_dev - Check if device is linked to an upper device
4863  * @dev: device
4864  * @upper_dev: upper device to check
4865  *
4866  * Find out if a device is linked to specified upper device and return true
4867  * in case it is. Note that this checks only immediate upper device,
4868  * not through a complete stack of devices. The caller must hold the RTNL lock.
4869  */
4870 bool netdev_has_upper_dev(struct net_device *dev,
4871                           struct net_device *upper_dev)
4872 {
4873         ASSERT_RTNL();
4874 
4875         return __netdev_find_adj(dev, upper_dev, &dev->all_adj_list.upper);
4876 }
4877 EXPORT_SYMBOL(netdev_has_upper_dev);
4878 
4879 /**
4880  * netdev_has_any_upper_dev - Check if device is linked to some device
4881  * @dev: device
4882  *
4883  * Find out if a device is linked to an upper device and return true in case
4884  * it is. The caller must hold the RTNL lock.
4885  */
4886 static bool netdev_has_any_upper_dev(struct net_device *dev)
4887 {
4888         ASSERT_RTNL();
4889 
4890         return !list_empty(&dev->all_adj_list.upper);
4891 }
4892 
4893 /**
4894  * netdev_master_upper_dev_get - Get master upper device
4895  * @dev: device
4896  *
4897  * Find a master upper device and return pointer to it or NULL in case
4898  * it's not there. The caller must hold the RTNL lock.
4899  */
4900 struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
4901 {
4902         struct netdev_adjacent *upper;
4903 
4904         ASSERT_RTNL();
4905 
4906         if (list_empty(&dev->adj_list.upper))
4907                 return NULL;
4908 
4909         upper = list_first_entry(&dev->adj_list.upper,
4910                                  struct netdev_adjacent, list);
4911         if (likely(upper->master))
4912                 return upper->dev;
4913         return NULL;
4914 }
4915 EXPORT_SYMBOL(netdev_master_upper_dev_get);
4916 
4917 void *netdev_adjacent_get_private(struct list_head *adj_list)
4918 {
4919         struct netdev_adjacent *adj;
4920 
4921         adj = list_entry(adj_list, struct netdev_adjacent, list);
4922 
4923         return adj->private;
4924 }
4925 EXPORT_SYMBOL(netdev_adjacent_get_private);
4926 
4927 /**
4928  * netdev_upper_get_next_dev_rcu - Get the next dev from upper list
4929  * @dev: device
4930  * @iter: list_head ** of the current position
4931  *
4932  * Gets the next device from the dev's upper list, starting from iter
4933  * position. The caller must hold RCU read lock.
4934  */
4935 struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
4936                                                  struct list_head **iter)
4937 {
4938         struct netdev_adjacent *upper;
4939 
4940         WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
4941 
4942         upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
4943 
4944         if (&upper->list == &dev->adj_list.upper)
4945                 return NULL;
4946 
4947         *iter = &upper->list;
4948 
4949         return upper->dev;
4950 }
4951 EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu);
4952 
4953 /**
4954  * netdev_all_upper_get_next_dev_rcu - Get the next dev from upper list
4955  * @dev: device
4956  * @iter: list_head ** of the current position
4957  *
4958  * Gets the next device from the dev's upper list, starting from iter
4959  * position. The caller must hold RCU read lock.
4960  */
4961 struct net_device *netdev_all_upper_get_next_dev_rcu(struct net_device *dev,
4962                                                      struct list_head **iter)
4963 {
4964         struct netdev_adjacent *upper;
4965 
4966         WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
4967 
4968         upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
4969 
4970         if (&upper->list == &dev->all_adj_list.upper)
4971                 return NULL;
4972 
4973         *iter = &upper->list;
4974 
4975         return upper->dev;
4976 }
4977 EXPORT_SYMBOL(netdev_all_upper_get_next_dev_rcu);
4978 
4979 /**
4980  * netdev_lower_get_next_private - Get the next ->private from the
4981  *                                 lower neighbour list
4982  * @dev: device
4983  * @iter: list_head ** of the current position
4984  *
4985  * Gets the next netdev_adjacent->private from the dev's lower neighbour
4986  * list, starting from iter position. The caller must hold either hold the
4987  * RTNL lock or its own locking that guarantees that the neighbour lower
4988  * list will remain unchainged.
4989  */
4990 void *netdev_lower_get_next_private(struct net_device *dev,
4991                                     struct list_head **iter)
4992 {
4993         struct netdev_adjacent *lower;
4994 
4995         lower = list_entry(*iter, struct netdev_adjacent, list);
4996 
4997         if (&lower->list == &dev->adj_list.lower)
4998                 return NULL;
4999 
5000         *iter = lower->list.next;
5001 
5002         return lower->private;
5003 }
5004 EXPORT_SYMBOL(netdev_lower_get_next_private);
5005 
5006 /**
5007  * netdev_lower_get_next_private_rcu - Get the next ->private from the
5008  *                                     lower neighbour list, RCU
5009  *                                     variant
5010  * @dev: device
5011  * @iter: list_head ** of the current position
5012  *
5013  * Gets the next netdev_adjacent->private from the dev's lower neighbour
5014  * list, starting from iter position. The caller must hold RCU read lock.
5015  */
5016 void *netdev_lower_get_next_private_rcu(struct net_device *dev,
5017                                         struct list_head **iter)
5018 {
5019         struct netdev_adjacent *lower;
5020 
5021         WARN_ON_ONCE(!rcu_read_lock_held());
5022 
5023         lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5024 
5025         if (&lower->list == &dev->adj_list.lower)
5026                 return NULL;
5027 
5028         *iter = &lower->list;
5029 
5030         return lower->private;
5031 }
5032 EXPORT_SYMBOL(netdev_lower_get_next_private_rcu);
5033 
5034 /**
5035  * netdev_lower_get_next - Get the next device from the lower neighbour
5036  *                         list
5037  * @dev: device
5038  * @iter: list_head ** of the current position
5039  *
5040  * Gets the next netdev_adjacent from the dev's lower neighbour
5041  * list, starting from iter position. The caller must hold RTNL lock or
5042  * its own locking that guarantees that the neighbour lower
5043  * list will remain unchainged.
5044  */
5045 void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter)
5046 {
5047         struct netdev_adjacent *lower;
5048 
5049         lower = list_entry((*iter)->next, struct netdev_adjacent, list);
5050 
5051         if (&lower->list == &dev->adj_list.lower)
5052                 return NULL;
5053 
5054         *iter = &lower->list;
5055 
5056         return lower->dev;
5057 }
5058 EXPORT_SYMBOL(netdev_lower_get_next);
5059 
5060 /**
5061  * netdev_lower_get_first_private_rcu - Get the first ->private from the
5062  *                                     lower neighbour list, RCU
5063  *                                     variant
5064  * @dev: device
5065  *
5066  * Gets the first netdev_adjacent->private from the dev's lower neighbour
5067  * list. The caller must hold RCU read lock.
5068  */
5069 void *netdev_lower_get_first_private_rcu(struct net_device *dev)
5070 {
5071         struct netdev_adjacent *lower;
5072 
5073         lower = list_first_or_null_rcu(&dev->adj_list.lower,
5074                         struct netdev_adjacent, list);
5075         if (lower)
5076                 return lower->private;
5077         return NULL;
5078 }
5079 EXPORT_SYMBOL(netdev_lower_get_first_private_rcu);
5080 
5081 /**
5082  * netdev_master_upper_dev_get_rcu - Get master upper device
5083  * @dev: device
5084  *
5085  * Find a master upper device and return pointer to it or NULL in case
5086  * it's not there. The caller must hold the RCU read lock.
5087  */
5088 struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
5089 {
5090         struct netdev_adjacent *upper;
5091 
5092         upper = list_first_or_null_rcu(&dev->adj_list.upper,
5093                                        struct netdev_adjacent, list);
5094         if (upper && likely(upper->master))
5095                 return upper->dev;
5096         return NULL;
5097 }
5098 EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
5099 
5100 static int netdev_adjacent_sysfs_add(struct net_device *dev,
5101                               struct net_device *adj_dev,
5102                               struct list_head *dev_list)
5103 {
5104         char linkname[IFNAMSIZ+7];
5105         sprintf(linkname, dev_list == &dev->adj_list.upper ?
5106                 "upper_%s" : "lower_%s", adj_dev->name);
5107         return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj),
5108                                  linkname);
5109 }
5110 static void netdev_adjacent_sysfs_del(struct net_device *dev,
5111                                char *name,
5112                                struct list_head *dev_list)
5113 {
5114         char linkname[IFNAMSIZ+7];
5115         sprintf(linkname, dev_list == &dev->adj_list.upper ?
5116                 "upper_%s" : "lower_%s", name);
5117         sysfs_remove_link(&(dev->dev.kobj), linkname);
5118 }
5119 
5120 static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev,
5121                                                  struct net_device *adj_dev,
5122                                                  struct list_head *dev_list)
5123 {
5124         return (dev_list == &dev->adj_list.upper ||
5125                 dev_list == &dev->adj_list.lower) &&
5126                 net_eq(dev_net(dev), dev_net(adj_dev));
5127 }
5128 
5129 static int __netdev_adjacent_dev_insert(struct net_device *dev,
5130                                         struct net_device *adj_dev,
5131                                         struct list_head *dev_list,
5132                                         void *private, bool master)
5133 {
5134         struct netdev_adjacent *adj;
5135         int ret;
5136 
5137         adj = __netdev_find_adj(dev, adj_dev, dev_list);
5138 
5139         if (adj) {
5140                 adj->ref_nr++;
5141                 return 0;
5142         }
5143 
5144         adj = kmalloc(sizeof(*adj), GFP_KERNEL);
5145         if (!adj)
5146                 return -ENOMEM;
5147 
5148         adj->dev = adj_dev;
5149         adj->master = master;
5150         adj->ref_nr = 1;
5151         adj->private = private;
5152         dev_hold(adj_dev);
5153 
5154         pr_debug("dev_hold for %s, because of link added from %s to %s\n",
5155                  adj_dev->name, dev->name, adj_dev->name);
5156 
5157         if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) {
5158                 ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list);
5159                 if (ret)
5160                         goto free_adj;
5161         }
5162 
5163         /* Ensure that master link is always the first item in list. */
5164         if (master) {
5165                 ret = sysfs_create_link(&(dev->dev.kobj),
5166                                         &(adj_dev->dev.kobj), "master");
5167                 if (ret)
5168                         goto remove_symlinks;
5169 
5170                 list_add_rcu(&adj->list, dev_list);
5171         } else {
5172                 list_add_tail_rcu(&adj->list, dev_list);
5173         }
5174 
5175         return 0;
5176 
5177 remove_symlinks:
5178         if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
5179                 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
5180 free_adj:
5181         kfree(adj);
5182         dev_put(adj_dev);
5183 
5184         return ret;
5185 }
5186 
5187 static void __netdev_adjacent_dev_remove(struct net_device *dev,
5188                                          struct net_device *adj_dev,
5189                                          struct list_head *dev_list)
5190 {
5191         struct netdev_adjacent *adj;
5192 
5193         adj = __netdev_find_adj(dev, adj_dev, dev_list);
5194 
5195         if (!adj) {
5196                 pr_err("tried to remove device %s from %s\n",
5197                        dev->name, adj_dev->name);
5198                 BUG();
5199         }
5200 
5201         if (adj->ref_nr > 1) {
5202                 pr_debug("%s to %s ref_nr-- = %d\n", dev->name, adj_dev->name,
5203                          adj->ref_nr-1);
5204                 adj->ref_nr--;
5205                 return;
5206         }
5207 
5208         if (adj->master)
5209                 sysfs_remove_link(&(dev->dev.kobj), "master");
5210 
5211         if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
5212                 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
5213 
5214         list_del_rcu(&adj->list);
5215         pr_debug("dev_put for %s, because link removed from %s to %s\n",
5216                  adj_dev->name, dev->name, adj_dev->name);
5217         dev_put(adj_dev);
5218         kfree_rcu(adj, rcu);
5219 }
5220 
5221 static int __netdev_adjacent_dev_link_lists(struct net_device *dev,
5222                                             struct net_device *upper_dev,
5223                                             struct list_head *up_list,
5224                                             struct list_head *down_list,
5225                                             void *private, bool master)
5226 {
5227         int ret;
5228 
5229         ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list, private,
5230                                            master);
5231         if (ret)
5232                 return ret;
5233 
5234         ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list, private,
5235                                            false);
5236         if (ret) {
5237                 __netdev_adjacent_dev_remove(dev, upper_dev, up_list);
5238                 return ret;
5239         }
5240 
5241         return 0;
5242 }
5243 
5244 static int __netdev_adjacent_dev_link(struct net_device *dev,
5245                                       struct net_device *upper_dev)
5246 {
5247         return __netdev_adjacent_dev_link_lists(dev, upper_dev,
5248                                                 &dev->all_adj_list.upper,
5249                                                 &upper_dev->all_adj_list.lower,
5250                                                 NULL, false);
5251 }
5252 
5253 static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev,
5254                                                struct net_device *upper_dev,
5255                                                struct list_head *up_list,
5256                                                struct list_head *down_list)
5257 {
5258         __netdev_adjacent_dev_remove(dev, upper_dev, up_list);
5259         __netdev_adjacent_dev_remove(upper_dev, dev, down_list);
5260 }
5261 
5262 static void __netdev_adjacent_dev_unlink(struct net_device *dev,
5263                                          struct net_device *upper_dev)
5264 {
5265         __netdev_adjacent_dev_unlink_lists(dev, upper_dev,
5266                                            &dev->all_adj_list.upper,
5267                                            &upper_dev->all_adj_list.lower);
5268 }
5269 
5270 static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev,
5271                                                 struct net_device *upper_dev,
5272                                                 void *private, bool master)
5273 {
5274         int ret = __netdev_adjacent_dev_link(dev, upper_dev);
5275 
5276         if (ret)
5277                 return ret;
5278 
5279         ret = __netdev_adjacent_dev_link_lists(dev, upper_dev,
5280                                                &dev->adj_list.upper,
5281                                                &upper_dev->adj_list.lower,
5282                                                private, master);
5283         if (ret) {
5284                 __netdev_adjacent_dev_unlink(dev, upper_dev);
5285                 return ret;
5286         }
5287 
5288         return 0;
5289 }
5290 
5291 static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev,
5292                                                    struct net_device *upper_dev)
5293 {
5294         __netdev_adjacent_dev_unlink(dev, upper_dev);
5295         __netdev_adjacent_dev_unlink_lists(dev, upper_dev,
5296                                            &dev->adj_list.upper,
5297                                            &upper_dev->adj_list.lower);
5298 }
5299 
5300 static int __netdev_upper_dev_link(struct net_device *dev,
5301                                    struct net_device *upper_dev, bool master,
5302                                    void *private)
5303 {
5304         struct netdev_adjacent *i, *j, *to_i, *to_j;
5305         int ret = 0;
5306 
5307         ASSERT_RTNL();
5308 
5309         if (dev == upper_dev)
5310                 return -EBUSY;
5311 
5312         /* To prevent loops, check if dev is not upper device to upper_dev. */
5313         if (__netdev_find_adj(upper_dev, dev, &upper_dev->all_adj_list.upper))
5314                 return -EBUSY;
5315 
5316         if (__netdev_find_adj(dev, upper_dev, &dev->adj_list.upper))
5317                 return -EEXIST;
5318 
5319         if (master && netdev_master_upper_dev_get(dev))
5320                 return -EBUSY;
5321 
5322         ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, private,
5323                                                    master);
5324         if (ret)
5325                 return ret;
5326 
5327         /* Now that we linked these devs, make all the upper_dev's
5328          * all_adj_list.upper visible to every dev's all_adj_list.lower an
5329          * versa, and don't forget the devices itself. All of these
5330          * links are non-neighbours.
5331          */
5332         list_for_each_entry(i, &dev->all_adj_list.lower, list) {
5333                 list_for_each_entry(j, &upper_dev->all_adj_list.upper, list) {
5334                         pr_debug("Interlinking %s with %s, non-neighbour\n",
5335                                  i->dev->name, j->dev->name);
5336                         ret = __netdev_adjacent_dev_link(i->dev, j->dev);
5337                         if (ret)
5338                                 goto rollback_mesh;
5339                 }
5340         }
5341 
5342         /* add dev to every upper_dev's upper device */
5343         list_for_each_entry(i, &upper_dev->all_adj_list.upper, list) {
5344                 pr_debug("linking %s's upper device %s with %s\n",
5345                          upper_dev->name, i->dev->name, dev->name);
5346                 ret = __netdev_adjacent_dev_link(dev, i->dev);
5347                 if (ret)
5348                         goto rollback_upper_mesh;
5349         }
5350 
5351         /* add upper_dev to every dev's lower device */
5352         list_for_each_entry(i, &dev->all_adj_list.lower, list) {
5353                 pr_debug("linking %s's lower device %s with %s\n", dev->name,
5354                          i->dev->name, upper_dev->name);
5355                 ret = __netdev_adjacent_dev_link(i->dev, upper_dev);
5356                 if (ret)
5357                         goto rollback_lower_mesh;
5358         }
5359 
5360         call_netdevice_notifiers(NETDEV_CHANGEUPPER, dev);
5361         return 0;
5362 
5363 rollback_lower_mesh:
5364         to_i = i;
5365         list_for_each_entry(i, &dev->all_adj_list.lower, list) {
5366                 if (i == to_i)
5367                         break;
5368                 __netdev_adjacent_dev_unlink(i->dev, upper_dev);
5369         }
5370 
5371         i = NULL;
5372 
5373 rollback_upper_mesh:
5374         to_i = i;
5375         list_for_each_entry(i, &upper_dev->all_adj_list.upper, list) {
5376                 if (i == to_i)
5377                         break;
5378                 __netdev_adjacent_dev_unlink(dev, i->dev);
5379         }
5380 
5381         i = j = NULL;
5382 
5383 rollback_mesh:
5384         to_i = i;
5385         to_j = j;
5386         list_for_each_entry(i, &dev->all_adj_list.lower, list) {
5387                 list_for_each_entry(j, &upper_dev->all_adj_list.upper, list) {
5388                         if (i == to_i && j == to_j)
5389                                 break;
5390                         __netdev_adjacent_dev_unlink(i->dev, j->dev);
5391                 }
5392                 if (i == to_i)
5393                         break;
5394         }
5395 
5396         __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
5397 
5398         return ret;
5399 }
5400 
5401 /**
5402  * netdev_upper_dev_link - Add a link to the upper device
5403  * @dev: device
5404  * @upper_dev: new upper device
5405  *
5406  * Adds a link to device which is upper to this one. The caller must hold
5407  * the RTNL lock. On a failure a negative errno code is returned.
5408  * On success the reference counts are adjusted and the function
5409  * returns zero.
5410  */
5411 int netdev_upper_dev_link(struct net_device *dev,
5412                           struct net_device *upper_dev)
5413 {
5414         return __netdev_upper_dev_link(dev, upper_dev, false, NULL);
5415 }
5416 EXPORT_SYMBOL(netdev_upper_dev_link);
5417 
5418 /**
5419  * netdev_master_upper_dev_link - Add a master link to the upper device
5420  * @dev: device
5421  * @upper_dev: new upper device
5422  *
5423  * Adds a link to device which is upper to this one. In this case, only
5424  * one master upper device can be linked, although other non-master devices
5425  * might be linked as well. The caller must hold the RTNL lock.
5426  * On a failure a negative errno code is returned. On success the reference
5427  * counts are adjusted and the function returns zero.
5428  */
5429 int netdev_master_upper_dev_link(struct net_device *dev,
5430                                  struct net_device *upper_dev)
5431 {
5432         return __netdev_upper_dev_link(dev, upper_dev, true, NULL);
5433 }
5434 EXPORT_SYMBOL(netdev_master_upper_dev_link);
5435 
5436 int netdev_master_upper_dev_link_private(struct net_device *dev,
5437                                          struct net_device *upper_dev,
5438                                          void *private)
5439 {
5440         return __netdev_upper_dev_link(dev, upper_dev, true, private);
5441 }
5442 EXPORT_SYMBOL(netdev_master_upper_dev_link_private);
5443 
5444 /**
5445  * netdev_upper_dev_unlink - Removes a link to upper device
5446  * @dev: device
5447  * @upper_dev: new upper device
5448  *
5449  * Removes a link to device which is upper to this one. The caller must hold
5450  * the RTNL lock.
5451  */
5452 void netdev_upper_dev_unlink(struct net_device *dev,
5453                              struct net_device *upper_dev)
5454 {
5455         struct netdev_adjacent *i, *j;
5456         ASSERT_RTNL();
5457 
5458         __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
5459 
5460         /* Here is the tricky part. We must remove all dev's lower
5461          * devices from all upper_dev's upper devices and vice
5462          * versa, to maintain the graph relationship.
5463          */
5464         list_for_each_entry(i, &dev->all_adj_list.lower, list)
5465                 list_for_each_entry(j, &upper_dev->all_adj_list.upper, list)
5466                         __netdev_adjacent_dev_unlink(i->dev, j->dev);
5467 
5468         /* remove also the devices itself from lower/upper device
5469          * list
5470          */
5471         list_for_each_entry(i, &dev->all_adj_list.lower, list)
5472                 __netdev_adjacent_dev_unlink(i->dev, upper_dev);
5473 
5474         list_for_each_entry(i, &upper_dev->all_adj_list.upper, list)
5475                 __netdev_adjacent_dev_unlink(dev, i->dev);
5476 
5477         call_netdevice_notifiers(NETDEV_CHANGEUPPER, dev);
5478 }
5479 EXPORT_SYMBOL(netdev_upper_dev_unlink);
5480 
5481 /**
5482  * netdev_bonding_info_change - Dispatch event about slave change
5483  * @dev: device
5484  * @bonding_info: info to dispatch
5485  *
5486  * Send NETDEV_BONDING_INFO to netdev notifiers with info.
5487  * The caller must hold the RTNL lock.
5488  */
5489 void netdev_bonding_info_change(struct net_device *dev,
5490                                 struct netdev_bonding_info *bonding_info)
5491 {
5492         struct netdev_notifier_bonding_info     info;
5493 
5494         memcpy(&info.bonding_info, bonding_info,
5495                sizeof(struct netdev_bonding_info));
5496         call_netdevice_notifiers_info(NETDEV_BONDING_INFO, dev,
5497                                       &info.info);
5498 }
5499 EXPORT_SYMBOL(netdev_bonding_info_change);
5500 
5501 static void netdev_adjacent_add_links(struct net_device *dev)
5502 {
5503         struct netdev_adjacent *iter;
5504 
5505         struct net *net = dev_net(dev);
5506 
5507         list_for_each_entry(iter, &dev->adj_list.upper, list) {
5508                 if (!net_eq(net,dev_net(iter->dev)))
5509                         continue;
5510                 netdev_adjacent_sysfs_add(iter->dev, dev,
5511                                           &iter->dev->adj_list.lower);
5512                 netdev_adjacent_sysfs_add(dev, iter->dev,
5513                                           &dev->adj_list.upper);
5514         }
5515 
5516         list_for_each_entry(iter, &dev->adj_list.lower, list) {
5517                 if (!net_eq(net,dev_net(iter->dev)))
5518                         continue;
5519                 netdev_adjacent_sysfs_add(iter->dev, dev,
5520                                           &iter->dev->adj_list.upper);
5521                 netdev_adjacent_sysfs_add(dev, iter->dev,
5522                                           &dev->adj_list.lower);
5523         }
5524 }
5525 
5526 static void netdev_adjacent_del_links(struct net_device *dev)
5527 {
5528         struct netdev_adjacent *iter;
5529 
5530         struct net *net = dev_net(dev);
5531 
5532         list_for_each_entry(iter, &dev->adj_list.upper, list) {
5533                 if (!net_eq(net,dev_net(iter->dev)))
5534                         continue;
5535                 netdev_adjacent_sysfs_del(iter->dev, dev->name,
5536                                           &iter->dev->adj_list.lower);
5537                 netdev_adjacent_sysfs_del(dev, iter->dev->name,
5538                                           &dev->adj_list.upper);
5539         }
5540 
5541         list_for_each_entry(iter, &dev->adj_list.lower, list) {
5542                 if (!net_eq(net,dev_net(iter->dev)))
5543                         continue;
5544                 netdev_adjacent_sysfs_del(iter->dev, dev->name,
5545                                           &iter->dev->adj_list.upper);
5546                 netdev_adjacent_sysfs_del(dev, iter->dev->name,
5547                                           &dev->adj_list.lower);
5548         }
5549 }
5550 
5551 void netdev_adjacent_rename_links(struct net_device *dev, char *oldname)
5552 {
5553         struct netdev_adjacent *iter;
5554 
5555         struct net *net = dev_net(dev);
5556 
5557         list_for_each_entry(iter, &dev->adj_list.upper, list) {
5558                 if (!net_eq(net,dev_net(iter->dev)))
5559                         continue;
5560                 netdev_adjacent_sysfs_del(iter->dev, oldname,
5561                                           &iter->dev->adj_list.lower);
5562                 netdev_adjacent_sysfs_add(iter->dev, dev,
5563                                           &iter->dev->adj_list.lower);
5564         }
5565 
5566         list_for_each_entry(iter, &dev->adj_list.lower, list) {
5567                 if (!net_eq(net,dev_net(iter->dev)))
5568                         continue;
5569                 netdev_adjacent_sysfs_del(iter->dev, oldname,
5570                                           &iter->dev->adj_list.upper);
5571                 netdev_adjacent_sysfs_add(iter->dev, dev,
5572                                           &iter->dev->adj_list.upper);
5573         }
5574 }
5575 
5576 void *netdev_lower_dev_get_private(struct net_device *dev,
5577                                    struct net_device *lower_dev)
5578 {
5579         struct netdev_adjacent *lower;
5580 
5581         if (!lower_dev)
5582                 return NULL;
5583         lower = __netdev_find_adj(dev, lower_dev, &dev->adj_list.lower);
5584         if (!lower)
5585                 return NULL;
5586 
5587         return lower->private;
5588 }
5589 EXPORT_SYMBOL(netdev_lower_dev_get_private);
5590 
5591 
5592 int dev_get_nest_level(struct net_device *dev,
5593                        bool (*type_check)(struct net_device *dev))
5594 {
5595         struct net_device *lower = NULL;
5596         struct list_head *iter;
5597         int max_nest = -1;
5598         int nest;
5599 
5600         ASSERT_RTNL();
5601 
5602         netdev_for_each_lower_dev(dev, lower, iter) {
5603                 nest = dev_get_nest_level(lower, type_check);
5604                 if (max_nest < nest)
5605                         max_nest = nest;
5606         }
5607 
5608         if (type_check(dev))
5609                 max_nest++;
5610 
5611         return max_nest;
5612 }
5613 EXPORT_SYMBOL(dev_get_nest_level);
5614 
5615 static void dev_change_rx_flags(struct net_device *dev, int flags)
5616 {
5617         const struct net_device_ops *ops = dev->netdev_ops;
5618 
5619         if (ops->ndo_change_rx_flags)
5620                 ops->ndo_change_rx_flags(dev, flags);
5621 }
5622 
5623 static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify)
5624 {
5625         unsigned int old_flags = dev->flags;
5626         kuid_t uid;
5627         kgid_t gid;
5628 
5629         ASSERT_RTNL();
5630 
5631         dev->flags |= IFF_PROMISC;
5632         dev->promiscuity += inc;
5633         if (dev->promiscuity == 0) {
5634                 /*
5635                  * Avoid overflow.
5636                  * If inc causes overflow, untouch promisc and return error.
5637                  */
5638                 if (inc < 0)
5639                         dev->flags &= ~IFF_PROMISC;
5640                 else {
5641                         dev->promiscuity -= inc;
5642                         pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n",
5643                                 dev->name);
5644                         return -EOVERFLOW;
5645                 }
5646         }
5647         if (dev->flags != old_flags) {
5648                 pr_info("device %s %s promiscuous mode\n",
5649                         dev->name,
5650                         dev->flags & IFF_PROMISC ? "entered" : "left");
5651                 if (audit_enabled) {
5652                         current_uid_gid(&uid, &gid);
5653                         audit_log(current->audit_context, GFP_ATOMIC,
5654                                 AUDIT_ANOM_PROMISCUOUS,
5655                                 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
5656                                 dev->name, (dev->flags & IFF_PROMISC),
5657                                 (old_flags & IFF_PROMISC),
5658                                 from_kuid(&init_user_ns, audit_get_loginuid(current)),
5659                                 from_kuid(&init_user_ns, uid),
5660                                 from_kgid(&init_user_ns, gid),
5661                                 audit_get_sessionid(current));
5662                 }
5663 
5664                 dev_change_rx_flags(dev, IFF_PROMISC);
5665         }
5666         if (notify)
5667                 __dev_notify_flags(dev, old_flags, IFF_PROMISC);
5668         return 0;
5669 }
5670 
5671 /**
5672  *      dev_set_promiscuity     - update promiscuity count on a device
5673  *      @dev: device
5674  *      @inc: modifier
5675  *
5676  *      Add or remove promiscuity from a device. While the count in the device
5677  *      remains above zero the interface remains promiscuous. Once it hits zero
5678  *      the device reverts back to normal filtering operation. A negative inc
5679  *      value is used to drop promiscuity on the device.
5680  *      Return 0 if successful or a negative errno code on error.
5681  */
5682 int dev_set_promiscuity(struct net_device *dev, int inc)
5683 {
5684         unsigned int old_flags = dev->flags;
5685         int err;
5686 
5687         err = __dev_set_promiscuity(dev, inc, true);
5688         if (err < 0)
5689                 return err;
5690         if (dev->flags != old_flags)
5691                 dev_set_rx_mode(dev);
5692         return err;
5693 }
5694 EXPORT_SYMBOL(dev_set_promiscuity);
5695 
5696 static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify)
5697 {
5698         unsigned int old_flags = dev->flags, old_gflags = dev->gflags;
5699 
5700         ASSERT_RTNL();
5701 
5702         dev->flags |= IFF_ALLMULTI;
5703         dev->allmulti += inc;
5704         if (dev->allmulti == 0) {
5705                 /*
5706                  * Avoid overflow.
5707                  * If inc causes overflow, untouch allmulti and return error.
5708                  */
5709                 if (inc < 0)
5710                         dev->flags &= ~IFF_ALLMULTI;
5711                 else {
5712                         dev->allmulti -= inc;
5713                         pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n",
5714                                 dev->name);
5715                         return -EOVERFLOW;
5716                 }
5717         }
5718         if (dev->flags ^ old_flags) {
5719                 dev_change_rx_flags(dev, IFF_ALLMULTI);
5720                 dev_set_rx_mode(dev);
5721                 if (notify)
5722                         __dev_notify_flags(dev, old_flags,
5723                                            dev->gflags ^ old_gflags);
5724         }
5725         return 0;
5726 }
5727 
5728 /**
5729  *      dev_set_allmulti        - update allmulti count on a device
5730  *      @dev: device
5731  *      @inc: modifier
5732  *
5733  *      Add or remove reception of all multicast frames to a device. While the
5734  *      count in the device remains above zero the interface remains listening
5735  *      to all interfaces. Once it hits zero the device reverts back to normal
5736  *      filtering operation. A negative @inc value is used to drop the counter
5737  *      when releasing a resource needing all multicasts.
5738  *      Return 0 if successful or a negative errno code on error.
5739  */
5740 
5741 int dev_set_allmulti(struct net_device *dev, int inc)
5742 {
5743         return __dev_set_allmulti(dev, inc, true);
5744 }
5745 EXPORT_SYMBOL(dev_set_allmulti);
5746 
5747 /*
5748  *      Upload unicast and multicast address lists to device and
5749  *      configure RX filtering. When the device doesn't support unicast
5750  *      filtering it is put in promiscuous mode while unicast addresses
5751  *      are present.
5752  */
5753 void __dev_set_rx_mode(struct net_device *dev)
5754 {
5755         const struct net_device_ops *ops = dev->netdev_ops;
5756 
5757         /* dev_open will call this function so the list will stay sane. */
5758         if (!(dev->flags&IFF_UP))
5759                 return;
5760 
5761         if (!netif_device_present(dev))
5762                 return;
5763 
5764         if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
5765                 /* Unicast addresses changes may only happen under the rtnl,
5766                  * therefore calling __dev_set_promiscuity here is safe.
5767                  */
5768                 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
5769                         __dev_set_promiscuity(dev, 1, false);
5770                         dev->uc_promisc = true;
5771                 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
5772                         __dev_set_promiscuity(dev, -1, false);
5773                         dev->uc_promisc = false;
5774                 }
5775         }
5776 
5777         if (ops->ndo_set_rx_mode)
5778                 ops->ndo_set_rx_mode(dev);
5779 }
5780 
5781 void dev_set_rx_mode(struct net_device *dev)
5782 {
5783         netif_addr_lock_bh(dev);
5784         __dev_set_rx_mode(dev);
5785         netif_addr_unlock_bh(dev);
5786 }
5787 
5788 /**
5789  *      dev_get_flags - get flags reported to userspace
5790  *      @dev: device
5791  *
5792  *      Get the combination of flag bits exported through APIs to userspace.
5793  */
5794 unsigned int dev_get_flags(const struct net_device *dev)
5795 {
5796         unsigned int flags;
5797 
5798         flags = (dev->flags & ~(IFF_PROMISC |
5799                                 IFF_ALLMULTI |
5800                                 IFF_RUNNING |
5801                                 IFF_LOWER_UP |
5802                                 IFF_DORMANT)) |
5803                 (dev->gflags & (IFF_PROMISC |
5804                                 IFF_ALLMULTI));
5805 
5806         if (netif_running(dev)) {
5807                 if (netif_oper_up(dev))
5808                         flags |= IFF_RUNNING;
5809                 if (netif_carrier_ok(dev))
5810                         flags |= IFF_LOWER_UP;
5811                 if (netif_dormant(dev))
5812                         flags |= IFF_DORMANT;
5813         }
5814 
5815         return flags;
5816 }
5817 EXPORT_SYMBOL(dev_get_flags);
5818 
5819 int __dev_change_flags(struct net_device *dev, unsigned int flags)
5820 {
5821         unsigned int old_flags = dev->flags;
5822         int ret;
5823 
5824         ASSERT_RTNL();
5825 
5826         /*
5827          *      Set the flags on our device.
5828          */
5829 
5830         dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
5831                                IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
5832                                IFF_AUTOMEDIA)) |
5833                      (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
5834                                     IFF_ALLMULTI));
5835 
5836         /*
5837          *      Load in the correct multicast list now the flags have changed.
5838          */
5839 
5840         if ((old_flags ^ flags) & IFF_MULTICAST)
5841                 dev_change_rx_flags(dev, IFF_MULTICAST);
5842 
5843         dev_set_rx_mode(dev);
5844 
5845         /*
5846          *      Have we downed the interface. We handle IFF_UP ourselves
5847          *      according to user attempts to set it, rather than blindly
5848          *      setting it.
5849          */
5850 
5851         ret = 0;
5852         if ((old_flags ^ flags) & IFF_UP)
5853                 ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev);
5854 
5855         if ((flags ^ dev->gflags) & IFF_PROMISC) {
5856                 int inc = (flags & IFF_PROMISC) ? 1 : -1;
5857                 unsigned int old_flags = dev->flags;
5858 
5859                 dev->gflags ^= IFF_PROMISC;
5860 
5861                 if (__dev_set_promiscuity(dev, inc, false) >= 0)
5862                         if (dev->flags != old_flags)
5863                                 dev_set_rx_mode(dev);
5864         }
5865 
5866         /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
5867            is important. Some (broken) drivers set IFF_PROMISC, when
5868            IFF_ALLMULTI is requested not asking us and not reporting.
5869          */
5870         if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
5871                 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
5872 
5873                 dev->gflags ^= IFF_ALLMULTI;
5874                 __dev_set_allmulti(dev, inc, false);
5875         }
5876 
5877         return ret;
5878 }
5879 
5880 void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
5881                         unsigned int gchanges)
5882 {
5883         unsigned int changes = dev->flags ^ old_flags;
5884 
5885         if (gchanges)
5886                 rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC);
5887 
5888         if (changes & IFF_UP) {
5889                 if (dev->flags & IFF_UP)
5890                         call_netdevice_notifiers(NETDEV_UP, dev);
5891                 else
5892                         call_netdevice_notifiers(NETDEV_DOWN, dev);
5893         }
5894 
5895         if (dev->flags & IFF_UP &&
5896             (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) {
5897                 struct netdev_notifier_change_info change_info;
5898 
5899                 change_info.flags_changed = changes;
5900                 call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
5901                                               &change_info.info);
5902         }
5903 }
5904 
5905 /**
5906  *      dev_change_flags - change device settings
5907  *      @dev: device
5908  *      @flags: device state flags
5909  *
5910  *      Change settings on device based state flags. The flags are
5911  *      in the userspace exported format.
5912  */
5913 int dev_change_flags(struct net_device *dev, unsigned int flags)
5914 {
5915         int ret;
5916         unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags;
5917 
5918         ret = __dev_change_flags(dev, flags);
5919         if (ret < 0)
5920                 return ret;
5921 
5922         changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags);
5923         __dev_notify_flags(dev, old_flags, changes);
5924         return ret;
5925 }
5926 EXPORT_SYMBOL(dev_change_flags);
5927 
5928 static int __dev_set_mtu(struct net_device *dev, int new_mtu)
5929 {
5930         const struct net_device_ops *ops = dev->netdev_ops;
5931 
5932         if (ops->ndo_change_mtu)
5933                 return ops->ndo_change_mtu(dev, new_mtu);
5934 
5935         dev->mtu = new_mtu;
5936         return 0;
5937 }
5938 
5939 /**
5940  *      dev_set_mtu - Change maximum transfer unit
5941  *      @dev: device
5942  *      @new_mtu: new transfer unit
5943  *
5944  *      Change the maximum transfer size of the network device.
5945  */
5946 int dev_set_mtu(struct net_device *dev, int new_mtu)
5947 {
5948         int err, orig_mtu;
5949 
5950         if (new_mtu == dev->mtu)
5951                 return 0;
5952 
5953         /*      MTU must be positive.    */
5954         if (new_mtu < 0)
5955                 return -EINVAL;
5956 
5957         if (!netif_device_present(dev))
5958                 return -ENODEV;
5959 
5960         err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev);
5961         err = notifier_to_errno(err);
5962         if (err)
5963                 return err;
5964 
5965         orig_mtu = dev->mtu;
5966         err = __dev_set_mtu(dev, new_mtu);
5967 
5968         if (!err) {
5969                 err = call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
5970                 err = notifier_to_errno(err);
5971                 if (err) {
5972                         /* setting mtu back and notifying everyone again,
5973                          * so that they have a chance to revert changes.
5974                          */
5975                         __dev_set_mtu(dev, orig_mtu);
5976                         call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
5977                 }
5978         }
5979         return err;
5980 }
5981 EXPORT_SYMBOL(dev_set_mtu);
5982 
5983 /**
5984  *      dev_set_group - Change group this device belongs to
5985  *      @dev: device
5986  *      @new_group: group this device should belong to
5987  */
5988 void dev_set_group(struct net_device *dev, int new_group)
5989 {
5990         dev->group = new_group;
5991 }
5992 EXPORT_SYMBOL(dev_set_group);
5993 
5994 /**
5995  *      dev_set_mac_address - Change Media Access Control Address
5996  *      @dev: device
5997  *      @sa: new address
5998  *
5999  *      Change the hardware (MAC) address of the device
6000  */
6001 int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
6002 {
6003         const struct net_device_ops *ops = dev->netdev_ops;
6004         int err;
6005 
6006         if (!ops->ndo_set_mac_address)
6007                 return -EOPNOTSUPP;
6008         if (sa->sa_family != dev->type)
6009                 return -EINVAL;
6010         if (!netif_device_present(dev))
6011                 return -ENODEV;
6012         err = ops->ndo_set_mac_address(dev, sa);
6013         if (err)
6014                 return err;
6015         dev->addr_assign_type = NET_ADDR_SET;
6016         call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
6017         add_device_randomness(dev->dev_addr, dev->addr_len);
6018         return 0;
6019 }
6020 EXPORT_SYMBOL(dev_set_mac_address);
6021 
6022 /**
6023  *      dev_change_carrier - Change device carrier
6024  *      @dev: device
6025  *      @new_carrier: new value
6026  *
6027  *      Change device carrier
6028  */
6029 int dev_change_carrier(struct net_device *dev, bool new_carrier)
6030 {
6031         const struct net_device_ops *ops = dev->netdev_ops;
6032 
6033         if (!ops->ndo_change_carrier)
6034                 return -EOPNOTSUPP;
6035         if (!netif_device_present(dev))
6036                 return -ENODEV;
6037         return ops->ndo_change_carrier(dev, new_carrier);
6038 }
6039 EXPORT_SYMBOL(dev_change_carrier);
6040 
6041 /**
6042  *      dev_get_phys_port_id - Get device physical port ID
6043  *      @dev: device
6044  *      @ppid: port ID
6045  *
6046  *      Get device physical port ID
6047  */
6048 int dev_get_phys_port_id(struct net_device *dev,
6049                          struct netdev_phys_item_id *ppid)
6050 {
6051         const struct net_device_ops *ops = dev->netdev_ops;
6052 
6053         if (!ops->ndo_get_phys_port_id)
6054                 return -EOPNOTSUPP;
6055         return ops->ndo_get_phys_port_id(dev, ppid);
6056 }
6057 EXPORT_SYMBOL(dev_get_phys_port_id);
6058 
6059 /**
6060  *      dev_get_phys_port_name - Get device physical port name
6061  *      @dev: device
6062  *      @name: port name
6063  *
6064  *      Get device physical port name
6065  */
6066 int dev_get_phys_port_name(struct net_device *dev,
6067                            char *name, size_t len)
6068 {
6069         const struct net_device_ops *ops = dev->netdev_ops;
6070 
6071         if (!ops->ndo_get_phys_port_name)
6072                 return -EOPNOTSUPP;
6073         return ops->ndo_get_phys_port_name(dev, name, len);
6074 }
6075 EXPORT_SYMBOL(dev_get_phys_port_name);
6076 
6077 /**
6078  *      dev_new_index   -       allocate an ifindex
6079  *      @net: the applicable net namespace
6080  *
6081  *      Returns a suitable unique value for a new device interface
6082  *      number.  The caller must hold the rtnl semaphore or the
6083  *      dev_base_lock to be sure it remains unique.
6084  */
6085 static int dev_new_index(struct net *net)
6086 {
6087         int ifindex = net->ifindex;
6088         for (;;) {
6089                 if (++ifindex <= 0)
6090                         ifindex = 1;
6091                 if (!__dev_get_by_index(net, ifindex))
6092                         return net->ifindex = ifindex;
6093         }
6094 }
6095 
6096 /* Delayed registration/unregisteration */
6097 static LIST_HEAD(net_todo_list);
6098 DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq);
6099 
6100 static void net_set_todo(struct net_device *dev)
6101 {
6102         list_add_tail(&dev->todo_list, &net_todo_list);
6103         dev_net(dev)->dev_unreg_count++;
6104 }
6105 
6106 static void rollback_registered_many(struct list_head *head)
6107 {
6108         struct net_device *dev, *tmp;
6109         LIST_HEAD(close_head);
6110 
6111         BUG_ON(dev_boot_phase);
6112         ASSERT_RTNL();
6113 
6114         list_for_each_entry_safe(dev, tmp, head, unreg_list) {
6115                 /* Some devices call without registering
6116                  * for initialization unwind. Remove those
6117                  * devices and proceed with the remaining.
6118                  */
6119                 if (dev->reg_state == NETREG_UNINITIALIZED) {
6120                         pr_debug("unregister_netdevice: device %s/%p never was registered\n",
6121                                  dev->name, dev);
6122 
6123                         WARN_ON(1);
6124                         list_del(&dev->unreg_list);
6125                         continue;
6126                 }
6127                 dev->dismantle = true;
6128                 BUG_ON(dev->reg_state != NETREG_REGISTERED);
6129         }
6130 
6131         /* If device is running, close it first. */
6132         list_for_each_entry(dev, head, unreg_list)
6133                 list_add_tail(&dev->close_list, &close_head);
6134         dev_close_many(&close_head, true);
6135 
6136         list_for_each_entry(dev, head, unreg_list) {
6137                 /* And unlink it from device chain. */
6138                 unlist_netdevice(dev);
6139 
6140                 dev->reg_state = NETREG_UNREGISTERING;
6141                 on_each_cpu(flush_backlog, dev, 1);
6142         }
6143 
6144         synchronize_net();
6145 
6146         list_for_each_entry(dev, head, unreg_list) {
6147                 struct sk_buff *skb = NULL;
6148 
6149                 /* Shutdown queueing discipline. */
6150                 dev_shutdown(dev);
6151 
6152 
6153                 /* Notify protocols, that we are about to destroy
6154                    this device. They should clean all the things.
6155                 */
6156                 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
6157 
6158                 if (!dev->rtnl_link_ops ||
6159                     dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
6160                         skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U,
6161                                                      GFP_KERNEL);
6162 
6163                 /*
6164                  *      Flush the unicast and multicast chains
6165                  */
6166                 dev_uc_flush(dev);
6167                 dev_mc_flush(dev);
6168 
6169                 if (dev->netdev_ops->ndo_uninit)
6170                         dev->netdev_ops->ndo_uninit(dev);
6171 
6172                 if (skb)
6173                         rtmsg_ifinfo_send(skb, dev, GFP_KERNEL);
6174 
6175                 /* Notifier chain MUST detach us all upper devices. */
6176                 WARN_ON(netdev_has_any_upper_dev(dev));
6177 
6178                 /* Remove entries from kobject tree */
6179                 netdev_unregister_kobject(dev);
6180 #ifdef CONFIG_XPS
6181                 /* Remove XPS queueing entries */
6182                 netif_reset_xps_queues_gt(dev, 0);
6183 #endif
6184         }
6185 
6186         synchronize_net();
6187 
6188         list_for_each_entry(dev, head, unreg_list)
6189                 dev_put(dev);
6190 }
6191 
6192 static void rollback_registered(struct net_device *dev)
6193 {
6194         LIST_HEAD(single);
6195 
6196         list_add(&dev->unreg_list, &single);
6197         rollback_registered_many(&single);
6198         list_del(&single);
6199 }
6200 
6201 static netdev_features_t netdev_fix_features(struct net_device *dev,
6202         netdev_features_t features)
6203 {
6204         /* Fix illegal checksum combinations */
6205         if ((features & NETIF_F_HW_CSUM) &&
6206             (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
6207                 netdev_warn(dev, "mixed HW and IP checksum settings.\n");
6208                 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
6209         }
6210 
6211         /* TSO requires that SG is present as well. */
6212         if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
6213                 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
6214                 features &= ~NETIF_F_ALL_TSO;
6215         }
6216 
6217         if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) &&
6218                                         !(features & NETIF_F_IP_CSUM)) {
6219                 netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n");
6220                 features &= ~NETIF_F_TSO;
6221                 features &= ~NETIF_F_TSO_ECN;
6222         }
6223 
6224         if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) &&
6225                                          !(features & NETIF_F_IPV6_CSUM)) {
6226                 netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n");
6227                 features &= ~NETIF_F_TSO6;
6228         }
6229 
6230         /* TSO ECN requires that TSO is present as well. */
6231         if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
6232                 features &= ~NETIF_F_TSO_ECN;
6233 
6234         /* Software GSO depends on SG. */
6235         if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
6236                 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
6237                 features &= ~NETIF_F_GSO;
6238         }
6239 
6240         /* UFO needs SG and checksumming */
6241         if (features & NETIF_F_UFO) {
6242                 /* maybe split UFO into V4 and V6? */
6243                 if (!((features & NETIF_F_GEN_CSUM) ||
6244                     (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))
6245                             == (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
6246                         netdev_dbg(dev,
6247                                 "Dropping NETIF_F_UFO since no checksum offload features.\n");
6248                         features &= ~NETIF_F_UFO;
6249                 }
6250 
6251                 if (!(features & NETIF_F_SG)) {
6252                         netdev_dbg(dev,
6253                                 "Dropping NETIF_F_UFO since no NETIF_F_SG feature.\n");
6254                         features &= ~NETIF_F_UFO;
6255                 }
6256         }
6257 
6258 #ifdef CONFIG_NET_RX_BUSY_POLL
6259         if (dev->netdev_ops->ndo_busy_poll)
6260                 features |= NETIF_F_BUSY_POLL;
6261         else
6262 #endif
6263                 features &= ~NETIF_F_BUSY_POLL;
6264 
6265         return features;
6266 }
6267 
6268 int __netdev_update_features(struct net_device *dev)
6269 {
6270         netdev_features_t features;
6271         int err = 0;
6272 
6273         ASSERT_RTNL();
6274 
6275         features = netdev_get_wanted_features(dev);
6276 
6277         if (dev->netdev_ops->ndo_fix_features)
6278                 features = dev->netdev_ops->ndo_fix_features(dev, features);
6279 
6280         /* driver might be less strict about feature dependencies */
6281         features = netdev_fix_features(dev, features);
6282 
6283         if (dev->features == features)
6284                 return 0;
6285 
6286         netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
6287                 &dev->features, &features);
6288 
6289         if (dev->netdev_ops->ndo_set_features)
6290                 err = dev->netdev_ops->ndo_set_features(dev, features);
6291 
6292         if (unlikely(err < 0)) {
6293                 netdev_err(dev,
6294                         "set_features() failed (%d); wanted %pNF, left %pNF\n",
6295                         err, &features, &dev->features);
6296                 return -1;
6297         }
6298 
6299         if (!err)
6300                 dev->features = features;
6301 
6302         return 1;
6303 }
6304 
6305 /**
6306  *      netdev_update_features - recalculate device features
6307  *      @dev: the device to check
6308  *
6309  *      Recalculate dev->features set and send notifications if it
6310  *      has changed. Should be called after driver or hardware dependent
6311  *      conditions might have changed that influence the features.
6312  */
6313 void netdev_update_features(struct net_device *dev)
6314 {
6315         if (__netdev_update_features(dev))
6316                 netdev_features_change(dev);
6317 }
6318 EXPORT_SYMBOL(netdev_update_features);
6319 
6320 /**
6321  *      netdev_change_features - recalculate device features
6322  *      @dev: the device to check
6323  *
6324  *      Recalculate dev->features set and send notifications even
6325  *      if they have not changed. Should be called instead of
6326  *      netdev_update_features() if also dev->vlan_features might
6327  *      have changed to allow the changes to be propagated to stacked
6328  *      VLAN devices.
6329  */
6330 void netdev_change_features(struct net_device *dev)
6331 {
6332         __netdev_update_features(dev);
6333         netdev_features_change(dev);
6334 }
6335 EXPORT_SYMBOL(netdev_change_features);
6336 
6337 /**
6338  *      netif_stacked_transfer_operstate -      transfer operstate
6339  *      @rootdev: the root or lower level device to transfer state from
6340  *      @dev: the device to transfer operstate to
6341  *
6342  *      Transfer operational state from root to device. This is normally
6343  *      called when a stacking relationship exists between the root
6344  *      device and the device(a leaf device).
6345  */
6346 void netif_stacked_transfer_operstate(const struct net_device *rootdev,
6347                                         struct net_device *dev)
6348 {
6349         if (rootdev->operstate == IF_OPER_DORMANT)
6350                 netif_dormant_on(dev);
6351         else
6352                 netif_dormant_off(dev);
6353 
6354         if (netif_carrier_ok(rootdev)) {
6355                 if (!netif_carrier_ok(dev))
6356                         netif_carrier_on(dev);
6357         } else {
6358                 if (netif_carrier_ok(dev))
6359                         netif_carrier_off(dev);
6360         }
6361 }
6362 EXPORT_SYMBOL(netif_stacked_transfer_operstate);
6363 
6364 #ifdef CONFIG_SYSFS
6365 static int netif_alloc_rx_queues(struct net_device *dev)
6366 {
6367         unsigned int i, count = dev->num_rx_queues;
6368         struct netdev_rx_queue *rx;
6369         size_t sz = count * sizeof(*rx);
6370 
6371         BUG_ON(count < 1);
6372 
6373         rx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
6374         if (!rx) {
6375                 rx = vzalloc(sz);
6376                 if (!rx)
6377                         return -ENOMEM;
6378         }
6379         dev->_rx = rx;
6380 
6381         for (i = 0; i < count; i++)
6382                 rx[i].dev = dev;
6383         return 0;
6384 }
6385 #endif
6386 
6387 static void netdev_init_one_queue(struct net_device *dev,
6388                                   struct netdev_queue *queue, void *_unused)
6389 {
6390         /* Initialize queue lock */
6391         spin_lock_init(&queue->_xmit_lock);
6392         netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
6393         queue->xmit_lock_owner = -1;
6394         netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
6395         queue->dev = dev;
6396 #ifdef CONFIG_BQL
6397         dql_init(&queue->dql, HZ);
6398 #endif
6399 }
6400 
6401 static void netif_free_tx_queues(struct net_device *dev)
6402 {
6403         kvfree(dev->_tx);
6404 }
6405 
6406 static int netif_alloc_netdev_queues(struct net_device *dev)
6407 {
6408         unsigned int count = dev->num_tx_queues;
6409         struct netdev_queue *tx;
6410         size_t sz = count * sizeof(*tx);
6411 
6412         if (count < 1 || count > 0xffff)
6413                 return -EINVAL;
6414 
6415         tx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
6416         if (!tx) {
6417                 tx = vzalloc(sz);
6418                 if (!tx)
6419                         return -ENOMEM;
6420         }
6421         dev->_tx = tx;
6422 
6423         netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
6424         spin_lock_init(&dev->tx_global_lock);
6425 
6426         return 0;
6427 }
6428 
6429 void netif_tx_stop_all_queues(struct net_device *dev)
6430 {
6431         unsigned int i;
6432 
6433         for (i = 0; i < dev->num_tx_queues; i++) {
6434                 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
6435                 netif_tx_stop_queue(txq);
6436         }
6437 }
6438 EXPORT_SYMBOL(netif_tx_stop_all_queues);
6439 
6440 /**
6441  *      register_netdevice      - register a network device
6442  *      @dev: device to register
6443  *
6444  *      Take a completed network device structure and add it to the kernel
6445  *      interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
6446  *      chain. 0 is returned on success. A negative errno code is returned
6447  *      on a failure to set up the device, or if the name is a duplicate.
6448  *
6449  *      Callers must hold the rtnl semaphore. You may want
6450  *      register_netdev() instead of this.
6451  *
6452  *      BUGS:
6453  *      The locking appears insufficient to guarantee two parallel registers
6454  *      will not get the same name.
6455  */
6456 
6457 int register_netdevice(struct net_device *dev)
6458 {
6459         int ret;
6460         struct net *net = dev_net(dev);
6461 
6462         BUG_ON(dev_boot_phase);
6463         ASSERT_RTNL();
6464 
6465         might_sleep();
6466 
6467         /* When net_device's are persistent, this will be fatal. */
6468         BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
6469         BUG_ON(!net);
6470 
6471         spin_lock_init(&dev->addr_list_lock);
6472         netdev_set_addr_lockdep_class(dev);
6473 
6474         ret = dev_get_valid_name(net, dev, dev->name);
6475         if (ret < 0)
6476                 goto out;
6477 
6478         /* Init, if this function is available */
6479         if (dev->netdev_ops->ndo_init) {
6480                 ret = dev->netdev_ops->ndo_init(dev);
6481                 if (ret) {
6482                         if (ret > 0)
6483                                 ret = -EIO;
6484                         goto out;
6485                 }
6486         }
6487 
6488         if (((dev->hw_features | dev->features) &
6489              NETIF_F_HW_VLAN_CTAG_FILTER) &&
6490             (!dev->netdev_ops->ndo_vlan_rx_add_vid ||
6491              !dev->netdev_ops->ndo_vlan_rx_kill_vid)) {
6492                 netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n");
6493                 ret = -EINVAL;
6494                 goto err_uninit;
6495         }
6496 
6497         ret = -EBUSY;
6498         if (!dev->ifindex)
6499                 dev->ifindex = dev_new_index(net);
6500         else if (__dev_get_by_index(net, dev->ifindex))
6501                 goto err_uninit;
6502 
6503         /* Transfer changeable features to wanted_features and enable
6504          * software offloads (GSO and GRO).
6505          */
6506         dev->hw_features |= NETIF_F_SOFT_FEATURES;
6507         dev->features |= NETIF_F_SOFT_FEATURES;
6508         dev->wanted_features = dev->features & dev->hw_features;
6509 
6510         if (!(dev->flags & IFF_LOOPBACK)) {
6511                 dev->hw_features |= NETIF_F_NOCACHE_COPY;
6512         }
6513 
6514         /* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
6515          */
6516         dev->vlan_features |= NETIF_F_HIGHDMA;
6517 
6518         /* Make NETIF_F_SG inheritable to tunnel devices.
6519          */
6520         dev->hw_enc_features |= NETIF_F_SG;
6521 
6522         /* Make NETIF_F_SG inheritable to MPLS.
6523          */
6524         dev->mpls_features |= NETIF_F_SG;
6525 
6526         ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
6527         ret = notifier_to_errno(ret);
6528         if (ret)
6529                 goto err_uninit;
6530 
6531         ret = netdev_register_kobject(dev);
6532         if (ret)
6533                 goto err_uninit;
6534         dev->reg_state = NETREG_REGISTERED;
6535 
6536         __netdev_update_features(dev);
6537 
6538         /*
6539          *      Default initial state at registry is that the
6540          *      device is present.
6541          */
6542 
6543         set_bit(__LINK_STATE_PRESENT, &dev->state);
6544 
6545         linkwatch_init_dev(dev);
6546 
6547         dev_init_scheduler(dev);
6548         dev_hold(dev);
6549         list_netdevice(dev);
6550         add_device_randomness(dev->dev_addr, dev->addr_len);
6551 
6552         /* If the device has permanent device address, driver should
6553          * set dev_addr and also addr_assign_type should be set to
6554          * NET_ADDR_PERM (default value).
6555          */
6556         if (dev->addr_assign_type == NET_ADDR_PERM)
6557                 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
6558 
6559         /* Notify protocols, that a new device appeared. */
6560         ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
6561         ret = notifier_to_errno(ret);
6562         if (ret) {
6563                 rollback_registered(dev);
6564                 dev->reg_state = NETREG_UNREGISTERED;
6565         }
6566         /*
6567          *      Prevent userspace races by waiting until the network
6568          *      device is fully setup before sending notifications.
6569          */
6570         if (!dev->rtnl_link_ops ||
6571             dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
6572                 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
6573 
6574 out:
6575         return ret;
6576 
6577 err_uninit:
6578         if (dev->netdev_ops->ndo_uninit)
6579                 dev->netdev_ops->ndo_uninit(dev);
6580         goto out;
6581 }
6582 EXPORT_SYMBOL(register_netdevice);
6583 
6584 /**
6585  *      init_dummy_netdev       - init a dummy network device for NAPI
6586  *      @dev: device to init
6587  *
6588  *      This takes a network device structure and initialize the minimum
6589  *      amount of fields so it can be used to schedule NAPI polls without
6590  *      registering a full blown interface. This is to be used by drivers
6591  *      that need to tie several hardware interfaces to a single NAPI
6592  *      poll scheduler due to HW limitations.
6593  */
6594 int init_dummy_netdev(struct net_device *dev)
6595 {
6596         /* Clear everything. Note we don't initialize spinlocks
6597          * are they aren't supposed to be taken by any of the
6598          * NAPI code and this dummy netdev is supposed to be
6599          * only ever used for NAPI polls
6600          */
6601         memset(dev, 0, sizeof(struct net_device));
6602 
6603         /* make sure we BUG if trying to hit standard
6604          * register/unregister code path
6605          */
6606         dev->reg_state = NETREG_DUMMY;
6607 
6608         /* NAPI wants this */
6609         INIT_LIST_HEAD(&dev->napi_list);
6610 
6611         /* a dummy interface is started by default */
6612         set_bit(__LINK_STATE_PRESENT, &dev->state);
6613         set_bit(__LINK_STATE_START, &dev->state);
6614 
6615         /* Note : We dont allocate pcpu_refcnt for dummy devices,
6616          * because users of this 'device' dont need to change
6617          * its refcount.
6618          */
6619 
6620         return 0;
6621 }
6622 EXPORT_SYMBOL_GPL(init_dummy_netdev);
6623 
6624 
6625 /**
6626  *      register_netdev - register a network device
6627  *      @dev: device to register
6628  *
6629  *      Take a completed network device structure and add it to the kernel
6630  *      interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
6631  *      chain. 0 is returned on success. A negative errno code is returned
6632  *      on a failure to set up the device, or if the name is a duplicate.
6633  *
6634  *      This is a wrapper around register_netdevice that takes the rtnl semaphore
6635  *      and expands the device name if you passed a format string to
6636  *      alloc_netdev.
6637  */
6638 int register_netdev(struct net_device *dev)
6639 {
6640         int err;
6641 
6642         rtnl_lock();
6643         err = register_netdevice(dev);
6644         rtnl_unlock();
6645         return err;
6646 }
6647 EXPORT_SYMBOL(register_netdev);
6648 
6649 int netdev_refcnt_read(const struct net_device *dev)
6650 {
6651         int i, refcnt = 0;
6652 
6653         for_each_possible_cpu(i)
6654                 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
6655         return refcnt;
6656 }
6657 EXPORT_SYMBOL(netdev_refcnt_read);
6658 
6659 /**
6660  * netdev_wait_allrefs - wait until all references are gone.
6661  * @dev: target net_device
6662  *
6663  * This is called when unregistering network devices.
6664  *
6665  * Any protocol or device that holds a reference should register
6666  * for netdevice notification, and cleanup and put back the
6667  * reference if they receive an UNREGISTER event.
6668  * We can get stuck here if buggy protocols don't correctly
6669  * call dev_put.
6670  */
6671 static void netdev_wait_allrefs(struct net_device *dev)
6672 {
6673         unsigned long rebroadcast_time, warning_time;
6674         int refcnt;
6675 
6676         linkwatch_forget_dev(dev);
6677 
6678         rebroadcast_time = warning_time = jiffies;
6679         refcnt = netdev_refcnt_read(dev);
6680 
6681         while (refcnt != 0) {
6682                 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
6683                         rtnl_lock();
6684 
6685                         /* Rebroadcast unregister notification */
6686                         call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
6687 
6688                         __rtnl_unlock();
6689                         rcu_barrier();
6690                         rtnl_lock();
6691 
6692                         call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
6693                         if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
6694                                      &dev->state)) {
6695                                 /* We must not have linkwatch events
6696                                  * pending on unregister. If this
6697                                  * happens, we simply run the queue
6698                                  * unscheduled, resulting in a noop
6699                                  * for this device.
6700                                  */
6701                                 linkwatch_run_queue();
6702                         }
6703 
6704                         __rtnl_unlock();
6705 
6706                         rebroadcast_time = jiffies;
6707                 }
6708 
6709                 msleep(250);
6710 
6711                 refcnt = netdev_refcnt_read(dev);
6712 
6713                 if (time_after(jiffies, warning_time + 10 * HZ)) {
6714                         pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
6715                                  dev->name, refcnt);
6716                         warning_time = jiffies;
6717                 }
6718         }
6719 }
6720 
6721 /* The sequence is:
6722  *
6723  *      rtnl_lock();
6724  *      ...
6725  *      register_netdevice(x1);
6726  *      register_netdevice(x2);
6727  *      ...
6728  *      unregister_netdevice(y1);
6729  *      unregister_netdevice(y2);
6730  *      ...
6731  *      rtnl_unlock();
6732  *      free_netdev(y1);
6733  *      free_netdev(y2);
6734  *
6735  * We are invoked by rtnl_unlock().
6736  * This allows us to deal with problems:
6737  * 1) We can delete sysfs objects which invoke hotplug
6738  *    without deadlocking with linkwatch via keventd.
6739  * 2) Since we run with the RTNL semaphore not held, we can sleep
6740  *    safely in order to wait for the netdev refcnt to drop to zero.
6741  *
6742  * We must not return until all unregister events added during
6743  * the interval the lock was held have been completed.
6744  */
6745 void netdev_run_todo(void)
6746 {
6747         struct list_head list;
6748 
6749         /* Snapshot list, allow later requests */
6750         list_replace_init(&net_todo_list, &list);
6751 
6752         __rtnl_unlock();
6753 
6754 
6755         /* Wait for rcu callbacks to finish before next phase */
6756         if (!list_empty(&list))
6757                 rcu_barrier();
6758 
6759         while (!list_empty(&list)) {
6760                 struct net_device *dev
6761                         = list_first_entry(&list, struct net_device, todo_list);
6762                 list_del(&dev->todo_list);
6763 
6764                 rtnl_lock();
6765                 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
6766                 __rtnl_unlock();
6767 
6768                 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
6769                         pr_err("network todo '%s' but state %d\n",
6770                                dev->name, dev->reg_state);
6771                         dump_stack();
6772                         continue;
6773                 }
6774 
6775                 dev->reg_state = NETREG_UNREGISTERED;
6776 
6777                 netdev_wait_allrefs(dev);
6778 
6779                 /* paranoia */
6780                 BUG_ON(netdev_refcnt_read(dev));
6781                 BUG_ON(!list_empty(&dev->ptype_all));
6782                 BUG_ON(!list_empty(&dev->ptype_specific));
6783                 WARN_ON(rcu_access_pointer(dev->ip_ptr));
6784                 WARN_ON(rcu_access_pointer(dev->ip6_ptr));
6785                 WARN_ON(dev->dn_ptr);
6786 
6787                 if (dev->destructor)
6788                         dev->destructor(dev);
6789 
6790                 /* Report a network device has been unregistered */
6791                 rtnl_lock();
6792                 dev_net(dev)->dev_unreg_count--;
6793                 __rtnl_unlock();
6794                 wake_up(&netdev_unregistering_wq);
6795 
6796                 /* Free network device */
6797                 kobject_put(&dev->dev.kobj);
6798         }
6799 }
6800 
6801 /* Convert net_device_stats to rtnl_link_stats64.  They have the same
6802  * fields in the same order, with only the type differing.
6803  */
6804 void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
6805                              const struct net_device_stats *netdev_stats)
6806 {
6807 #if BITS_PER_LONG == 64
6808         BUILD_BUG_ON(sizeof(*stats64) != sizeof(*netdev_stats));
6809         memcpy(stats64, netdev_stats, sizeof(*stats64));
6810 #else
6811         size_t i, n = sizeof(*stats64) / sizeof(u64);
6812         const unsigned long *src = (const unsigned long *)netdev_stats;
6813         u64 *dst = (u64 *)stats64;
6814 
6815         BUILD_BUG_ON(sizeof(*netdev_stats) / sizeof(unsigned long) !=
6816                      sizeof(*stats64) / sizeof(u64));
6817         for (i = 0; i < n; i++)
6818                 dst[i] = src[i];
6819 #endif
6820 }
6821 EXPORT_SYMBOL(netdev_stats_to_stats64);
6822 
6823 /**
6824  *      dev_get_stats   - get network device statistics
6825  *      @dev: device to get statistics from
6826  *      @storage: place to store stats
6827  *
6828  *      Get network statistics from device. Return @storage.
6829  *      The device driver may provide its own method by setting
6830  *      dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
6831  *      otherwise the internal statistics structure is used.
6832  */
6833 struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
6834                                         struct rtnl_link_stats64 *storage)
6835 {
6836         const struct net_device_ops *ops = dev->netdev_ops;
6837 
6838         if (ops->ndo_get_stats64) {
6839                 memset(storage, 0, sizeof(*storage));
6840                 ops->ndo_get_stats64(dev, storage);
6841         } else if (ops->ndo_get_stats) {
6842                 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
6843         } else {
6844                 netdev_stats_to_stats64(storage, &dev->stats);
6845         }
6846         storage->rx_dropped += atomic_long_read(&dev->rx_dropped);
6847         storage->tx_dropped += atomic_long_read(&dev->tx_dropped);
6848         return storage;
6849 }
6850 EXPORT_SYMBOL(dev_get_stats);
6851 
6852 struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
6853 {
6854         struct netdev_queue *queue = dev_ingress_queue(dev);
6855 
6856 #ifdef CONFIG_NET_CLS_ACT
6857         if (queue)
6858                 return queue;
6859         queue = kzalloc(sizeof(*queue), GFP_KERNEL);
6860         if (!queue)
6861                 return NULL;
6862         netdev_init_one_queue(dev, queue, NULL);
6863         RCU_INIT_POINTER(queue->qdisc, &noop_qdisc);
6864         queue->qdisc_sleeping = &noop_qdisc;
6865         rcu_assign_pointer(dev->ingress_queue, queue);
6866 #endif
6867         return queue;
6868 }
6869 
6870 static const struct ethtool_ops default_ethtool_ops;
6871 
6872 void netdev_set_default_ethtool_ops(struct net_device *dev,
6873                                     const struct ethtool_ops *ops)
6874 {
6875         if (dev->ethtool_ops == &default_ethtool_ops)
6876                 dev->ethtool_ops = ops;
6877 }
6878 EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops);
6879 
6880 void netdev_freemem(struct net_device *dev)
6881 {
6882         char *addr = (char *)dev - dev->padded;
6883 
6884         kvfree(addr);
6885 }
6886 
6887 /**
6888  *      alloc_netdev_mqs - allocate network device
6889  *      @sizeof_priv:           size of private data to allocate space for
6890  *      @name:                  device name format string
6891  *      @name_assign_type:      origin of device name
6892  *      @setup:                 callback to initialize device
6893  *      @txqs:                  the number of TX subqueues to allocate
6894  *      @rxqs:                  the number of RX subqueues to allocate
6895  *
6896  *      Allocates a struct net_device with private data area for driver use
6897  *      and performs basic initialization.  Also allocates subqueue structs
6898  *      for each queue on the device.
6899  */
6900 struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
6901                 unsigned char name_assign_type,
6902                 void (*setup)(struct net_device *),
6903                 unsigned int txqs, unsigned int rxqs)
6904 {
6905         struct net_device *dev;
6906         size_t alloc_size;
6907         struct net_device *p;
6908 
6909         BUG_ON(strlen(name) >= sizeof(dev->name));
6910 
6911         if (txqs < 1) {
6912                 pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
6913                 return NULL;
6914         }
6915 
6916 #ifdef CONFIG_SYSFS
6917         if (rxqs < 1) {
6918                 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
6919                 return NULL;
6920         }
6921 #endif
6922 
6923         alloc_size = sizeof(struct net_device);
6924         if (sizeof_priv) {
6925                 /* ensure 32-byte alignment of private area */
6926                 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
6927                 alloc_size += sizeof_priv;
6928         }
6929         /* ensure 32-byte alignment of whole construct */
6930         alloc_size += NETDEV_ALIGN - 1;
6931 
6932         p = kzalloc(alloc_size, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
6933         if (!p)
6934                 p = vzalloc(alloc_size);
6935         if (!p)
6936                 return NULL;
6937 
6938         dev = PTR_ALIGN(p, NETDEV_ALIGN);
6939         dev->padded = (char *)dev - (char *)p;
6940 
6941         dev->pcpu_refcnt = alloc_percpu(int);
6942         if (!dev->pcpu_refcnt)
6943                 goto free_dev;
6944 
6945         if (dev_addr_init(dev))
6946                 goto free_pcpu;
6947 
6948         dev_mc_init(dev);
6949         dev_uc_init(dev);
6950 
6951         dev_net_set(dev, &init_net);
6952 
6953         dev->gso_max_size = GSO_MAX_SIZE;
6954         dev->gso_max_segs = GSO_MAX_SEGS;
6955         dev->gso_min_segs = 0;
6956 
6957         INIT_LIST_HEAD(&dev->napi_list);
6958         INIT_LIST_HEAD(&dev->unreg_list);
6959         INIT_LIST_HEAD(&dev->close_list);
6960         INIT_LIST_HEAD(&dev->link_watch_list);
6961         INIT_LIST_HEAD(&dev->adj_list.upper);
6962         INIT_LIST_HEAD(&dev->adj_list.lower);
6963         INIT_LIST_HEAD(&dev->all_adj_list.upper);
6964         INIT_LIST_HEAD(&dev->all_adj_list.lower);
6965         INIT_LIST_HEAD(&dev->ptype_all);
6966         INIT_LIST_HEAD(&dev->ptype_specific);
6967         dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM;
6968         setup(dev);
6969 
6970         dev->num_tx_queues = txqs;
6971         dev->real_num_tx_queues = txqs;
6972         if (netif_alloc_netdev_queues(dev))
6973                 goto free_all;
6974 
6975 #ifdef CONFIG_SYSFS
6976         dev->num_rx_queues = rxqs;
6977         dev->real_num_rx_queues = rxqs;
6978         if (netif_alloc_rx_queues(dev))
6979                 goto free_all;
6980 #endif
6981 
6982         strcpy(dev->name, name);
6983         dev->name_assign_type = name_assign_type;
6984         dev->group = INIT_NETDEV_GROUP;
6985         if (!dev->ethtool_ops)
6986                 dev->ethtool_ops = &default_ethtool_ops;
6987 
6988         nf_hook_ingress_init(dev);
6989 
6990         return dev;
6991 
6992 free_all:
6993         free_netdev(dev);
6994         return NULL;
6995 
6996 free_pcpu:
6997         free_percpu(dev->pcpu_refcnt);
6998 free_dev:
6999         netdev_freemem(dev);
7000         return NULL;
7001 }
7002 EXPORT_SYMBOL(alloc_netdev_mqs);
7003 
7004 /**
7005  *      free_netdev - free network device
7006  *      @dev: device
7007  *
7008  *      This function does the last stage of destroying an allocated device
7009  *      interface. The reference to the device object is released.
7010  *      If this is the last reference then it will be freed.
7011  */
7012 void free_netdev(struct net_device *dev)
7013 {
7014         struct napi_struct *p, *n;
7015 
7016         netif_free_tx_queues(dev);
7017 #ifdef CONFIG_SYSFS
7018         kvfree(dev->_rx);
7019 #endif
7020 
7021         kfree(rcu_dereference_protected(dev->ingress_queue, 1));
7022 
7023         /* Flush device addresses */
7024         dev_addr_flush(dev);
7025 
7026         list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
7027                 netif_napi_del(p);
7028 
7029         free_percpu(dev->pcpu_refcnt);
7030         dev->pcpu_refcnt = NULL;
7031 
7032         /*  Compatibility with error handling in drivers */
7033         if (dev->reg_state == NETREG_UNINITIALIZED) {
7034                 netdev_freemem(dev);
7035                 return;
7036         }
7037 
7038         BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
7039         dev->reg_state = NETREG_RELEASED;
7040 
7041         /* will free via device release */
7042         put_device(&dev->dev);
7043 }
7044 EXPORT_SYMBOL(free_netdev);
7045 
7046 /**
7047  *      synchronize_net -  Synchronize with packet receive processing
7048  *
7049  *      Wait for packets currently being received to be done.
7050  *      Does not block later packets from starting.
7051  */
7052 void synchronize_net(void)
7053 {
7054         might_sleep();
7055         if (rtnl_is_locked())
7056                 synchronize_rcu_expedited();
7057         else
7058                 synchronize_rcu();
7059 }
7060 EXPORT_SYMBOL(synchronize_net);
7061 
7062 /**
7063  *      unregister_netdevice_queue - remove device from the kernel
7064  *      @dev: device
7065  *      @head: list
7066  *
7067  *      This function shuts down a device interface and removes it
7068  *      from the kernel tables.
7069  *      If head not NULL, device is queued to be unregistered later.
7070  *
7071  *      Callers must hold the rtnl semaphore.  You may want
7072  *      unregister_netdev() instead of this.
7073  */
7074 
7075 void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
7076 {
7077         ASSERT_RTNL();
7078 
7079         if (head) {
7080                 list_move_tail(&dev->unreg_list, head);
7081         } else {
7082                 rollback_registered(dev);
7083                 /* Finish processing unregister after unlock */
7084                 net_set_todo(dev);
7085         }
7086 }
7087 EXPORT_SYMBOL(unregister_netdevice_queue);
7088 
7089 /**
7090  *      unregister_netdevice_many - unregister many devices
7091  *      @head: list of devices
7092  *
7093  *  Note: As most callers use a stack allocated list_head,
7094  *  we force a list_del() to make sure stack wont be corrupted later.
7095  */
7096 void unregister_netdevice_many(struct list_head *head)
7097 {
7098         struct net_device *dev;
7099 
7100         if (!list_empty(head)) {
7101                 rollback_registered_many(head);
7102                 list_for_each_entry(dev, head, unreg_list)
7103                         net_set_todo(dev);
7104                 list_del(head);
7105         }
7106 }
7107 EXPORT_SYMBOL(unregister_netdevice_many);
7108 
7109 /**
7110  *      unregister_netdev - remove device from the kernel
7111  *      @dev: device
7112  *
7113  *      This function shuts down a device interface and removes it
7114  *      from the kernel tables.
7115  *
7116  *      This is just a wrapper for unregister_netdevice that takes
7117  *      the rtnl semaphore.  In general you want to use this and not
7118  *      unregister_netdevice.
7119  */
7120 void unregister_netdev(struct net_device *dev)
7121 {
7122         rtnl_lock();
7123         unregister_netdevice(dev);
7124         rtnl_unlock();
7125 }
7126 EXPORT_SYMBOL(unregister_netdev);
7127 
7128 /**
7129  *      dev_change_net_namespace - move device to different nethost namespace
7130  *      @dev: device
7131  *      @net: network namespace
7132  *      @pat: If not NULL name pattern to try if the current device name
7133  *            is already taken in the destination network namespace.
7134  *
7135  *      This function shuts down a device interface and moves it
7136  *      to a new network namespace. On success 0 is returned, on
7137  *      a failure a netagive errno code is returned.
7138  *
7139  *      Callers must hold the rtnl semaphore.
7140  */
7141 
7142 int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
7143 {
7144         int err;
7145 
7146         ASSERT_RTNL();
7147 
7148         /* Don't allow namespace local devices to be moved. */
7149         err = -EINVAL;
7150         if (dev->features & NETIF_F_NETNS_LOCAL)
7151                 goto out;
7152 
7153         /* Ensure the device has been registrered */
7154         if (dev->reg_state != NETREG_REGISTERED)
7155                 goto out;
7156 
7157         /* Get out if there is nothing todo */
7158         err = 0;
7159         if (net_eq(dev_net(dev), net))
7160                 goto out;
7161 
7162         /* Pick the destination device name, and ensure
7163          * we can use it in the destination network namespace.
7164          */
7165         err = -EEXIST;
7166         if (__dev_get_by_name(net, dev->name)) {
7167                 /* We get here if we can't use the current device name */
7168                 if (!pat)
7169                         goto out;
7170                 if (dev_get_valid_name(net, dev, pat) < 0)
7171                         goto out;
7172         }
7173 
7174         /*
7175          * And now a mini version of register_netdevice unregister_netdevice.
7176          */
7177 
7178         /* If device is running close it first. */
7179         dev_close(dev);
7180 
7181         /* And unlink it from device chain */
7182         err = -ENODEV;
7183         unlist_netdevice(dev);
7184 
7185         synchronize_net();
7186 
7187         /* Shutdown queueing discipline. */
7188         dev_shutdown(dev);
7189 
7190         /* Notify protocols, that we are about to destroy
7191            this device. They should clean all the things.
7192 
7193            Note that dev->reg_state stays at NETREG_REGISTERED.
7194            This is wanted because this way 8021q and macvlan know
7195            the device is just moving and can keep their slaves up.
7196         */
7197         call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
7198         rcu_barrier();
7199         call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
7200         rtmsg_ifinfo(RTM_DELLINK, dev, ~0U, GFP_KERNEL);
7201 
7202         /*
7203          *      Flush the unicast and multicast chains
7204          */
7205         dev_uc_flush(dev);
7206         dev_mc_flush(dev);
7207 
7208         /* Send a netdev-removed uevent to the old namespace */
7209         kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);
7210         netdev_adjacent_del_links(dev);
7211 
7212         /* Actually switch the network namespace */
7213         dev_net_set(dev, net);
7214 
7215         /* If there is an ifindex conflict assign a new one */
7216         if (__dev_get_by_index(net, dev->ifindex))
7217                 dev->ifindex = dev_new_index(net);
7218 
7219         /* Send a netdev-add uevent to the new namespace */
7220         kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
7221         netdev_adjacent_add_links(dev);
7222 
7223         /* Fixup kobjects */
7224         err = device_rename(&dev->dev, dev->name);
7225         WARN_ON(err);
7226 
7227         /* Add the device back in the hashes */
7228         list_netdevice(dev);
7229 
7230         /* Notify protocols, that a new device appeared. */
7231         call_netdevice_notifiers(NETDEV_REGISTER, dev);
7232 
7233         /*
7234          *      Prevent userspace races by waiting until the network
7235          *      device is fully setup before sending notifications.
7236          */
7237         rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
7238 
7239         synchronize_net();
7240         err = 0;
7241 out:
7242         return err;
7243 }
7244 EXPORT_SYMBOL_GPL(dev_change_net_namespace);
7245 
7246 static int dev_cpu_callback(struct notifier_block *nfb,
7247                             unsigned long action,
7248                             void *ocpu)
7249 {
7250         struct sk_buff **list_skb;
7251         struct sk_buff *skb;
7252         unsigned int cpu, oldcpu = (unsigned long)ocpu;
7253         struct softnet_data *sd, *oldsd;
7254 
7255         if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
7256                 return NOTIFY_OK;
7257 
7258         local_irq_disable();
7259         cpu = smp_processor_id();
7260         sd = &per_cpu(softnet_data, cpu);
7261         oldsd = &per_cpu(softnet_data, oldcpu);
7262 
7263         /* Find end of our completion_queue. */
7264         list_skb = &sd->completion_queue;
7265         while (*list_skb)
7266                 list_skb = &(*list_skb)->next;
7267         /* Append completion queue from offline CPU. */
7268         *list_skb = oldsd->completion_queue;
7269         oldsd->completion_queue = NULL;
7270 
7271         /* Append output queue from offline CPU. */
7272         if (oldsd->output_queue) {
7273                 *sd->output_queue_tailp = oldsd->output_queue;
7274                 sd->output_queue_tailp = oldsd->output_queue_tailp;
7275                 oldsd->output_queue = NULL;
7276                 oldsd->output_queue_tailp = &oldsd->output_queue;
7277         }
7278         /* Append NAPI poll list from offline CPU, with one exception :
7279          * process_backlog() must be called by cpu owning percpu backlog.
7280          * We properly handle process_queue & input_pkt_queue later.
7281          */
7282         while (!list_empty(&oldsd->poll_list)) {
7283                 struct napi_struct *napi = list_first_entry(&oldsd->poll_list,
7284                                                             struct napi_struct,
7285                                                             poll_list);
7286 
7287                 list_del_init(&napi->poll_list);
7288                 if (napi->poll == process_backlog)
7289                         napi->state = 0;
7290                 else
7291                         ____napi_schedule(sd, napi);
7292         }
7293 
7294         raise_softirq_irqoff(NET_TX_SOFTIRQ);
7295         local_irq_enable();
7296 
7297         /* Process offline CPU's input_pkt_queue */
7298         while ((skb = __skb_dequeue(&oldsd->process_queue))) {
7299                 netif_rx_ni(skb);
7300                 input_queue_head_incr(oldsd);
7301         }
7302         while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) {
7303                 netif_rx_ni(skb);
7304                 input_queue_head_incr(oldsd);
7305         }
7306 
7307         return NOTIFY_OK;
7308 }
7309 
7310 
7311 /**
7312  *      netdev_increment_features - increment feature set by one
7313  *      @all: current feature set
7314  *      @one: new feature set
7315  *      @mask: mask feature set
7316  *
7317  *      Computes a new feature set after adding a device with feature set
7318  *      @one to the master device with current feature set @all.  Will not
7319  *      enable anything that is off in @mask. Returns the new feature set.
7320  */
7321 netdev_features_t netdev_increment_features(netdev_features_t all,
7322         netdev_features_t one, netdev_features_t mask)
7323 {
7324         if (mask & NETIF_F_GEN_CSUM)
7325                 mask |= NETIF_F_ALL_CSUM;
7326         mask |= NETIF_F_VLAN_CHALLENGED;
7327 
7328         all |= one & (NETIF_F_ONE_FOR_ALL|NETIF_F_ALL_CSUM) & mask;
7329         all &= one | ~NETIF_F_ALL_FOR_ALL;
7330 
7331         /* If one device supports hw checksumming, set for all. */
7332         if (all & NETIF_F_GEN_CSUM)
7333                 all &= ~(NETIF_F_ALL_CSUM & ~NETIF_F_GEN_CSUM);
7334 
7335         return all;
7336 }
7337 EXPORT_SYMBOL(netdev_increment_features);
7338 
7339 static struct hlist_head * __net_init netdev_create_hash(void)
7340 {
7341         int i;
7342         struct hlist_head *hash;
7343 
7344         hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
7345         if (hash != NULL)
7346                 for (i = 0; i < NETDEV_HASHENTRIES; i++)
7347                         INIT_HLIST_HEAD(&hash[i]);
7348 
7349         return hash;
7350 }
7351 
7352 /* Initialize per network namespace state */
7353 static int __net_init netdev_init(struct net *net)
7354 {
7355         if (net != &init_net)
7356                 INIT_LIST_HEAD(&net->dev_base_head);
7357 
7358         net->dev_name_head = netdev_create_hash();
7359         if (net->dev_name_head == NULL)
7360                 goto err_name;
7361 
7362         net->dev_index_head = netdev_create_hash();
7363         if (net->dev_index_head == NULL)
7364                 goto err_idx;
7365 
7366         return 0;
7367 
7368 err_idx:
7369         kfree(net->dev_name_head);
7370 err_name:
7371         return -ENOMEM;
7372 }
7373 
7374 /**
7375  *      netdev_drivername - network driver for the device
7376  *      @dev: network device
7377  *
7378  *      Determine network driver for device.
7379  */
7380 const char *netdev_drivername(const struct net_device *dev)
7381 {
7382         const struct device_driver *driver;
7383         const struct device *parent;
7384         const char *empty = "";
7385 
7386         parent = dev->dev.parent;
7387         if (!parent)
7388                 return empty;
7389 
7390         driver = parent->driver;
7391         if (driver && driver->name)
7392                 return driver->name;
7393         return empty;
7394 }
7395 
7396 static void __netdev_printk(const char *level, const struct net_device *dev,
7397                             struct va_format *vaf)
7398 {
7399         if (dev && dev->dev.parent) {
7400                 dev_printk_emit(level[1] - '',
7401                                 dev->dev.parent,
7402                                 "%s %s %s%s: %pV",
7403                                 dev_driver_string(dev->dev.parent),
7404                                 dev_name(dev->dev.parent),
7405                                 netdev_name(dev), netdev_reg_state(dev),
7406                                 vaf);
7407         } else if (dev) {
7408                 printk("%s%s%s: %pV",
7409                        level, netdev_name(dev), netdev_reg_state(dev), vaf);
7410         } else {
7411                 printk("%s(NULL net_device): %pV", level, vaf);
7412         }
7413 }
7414 
7415 void netdev_printk(const char *level, const struct net_device *dev,
7416                    const char *format, ...)
7417 {
7418         struct va_format vaf;
7419         va_list args;
7420 
7421         va_start(args, format);
7422 
7423         vaf.fmt = format;
7424         vaf.va = &args;
7425 
7426         __netdev_printk(level, dev, &vaf);
7427 
7428         va_end(args);
7429 }
7430 EXPORT_SYMBOL(netdev_printk);
7431 
7432 #define define_netdev_printk_level(func, level)                 \
7433 void func(const struct net_device *dev, const char *fmt, ...)   \
7434 {                                                               \
7435         struct va_format vaf;                                   \
7436         va_list args;                                           \
7437                                                                 \
7438         va_start(args, fmt);                                    \
7439                                                                 \
7440         vaf.fmt = fmt;                                          \
7441         vaf.va = &args;                                         \
7442                                                                 \
7443         __netdev_printk(level, dev, &vaf);                      \
7444                                                                 \
7445         va_end(args);                                           \
7446 }                                                               \
7447 EXPORT_SYMBOL(func);
7448 
7449 define_netdev_printk_level(netdev_emerg, KERN_EMERG);
7450 define_netdev_printk_level(netdev_alert, KERN_ALERT);
7451 define_netdev_printk_level(netdev_crit, KERN_CRIT);
7452 define_netdev_printk_level(netdev_err, KERN_ERR);
7453 define_netdev_printk_level(netdev_warn, KERN_WARNING);
7454 define_netdev_printk_level(netdev_notice, KERN_NOTICE);
7455 define_netdev_printk_level(netdev_info, KERN_INFO);
7456 
7457 static void __net_exit netdev_exit(struct net *net)
7458 {
7459         kfree(net->dev_name_head);
7460         kfree(net->dev_index_head);
7461 }
7462 
7463 static struct pernet_operations __net_initdata netdev_net_ops = {
7464         .init = netdev_init,
7465         .exit = netdev_exit,
7466 };
7467 
7468 static void __net_exit default_device_exit(struct net *net)
7469 {
7470         struct net_device *dev, *aux;
7471         /*
7472          * Push all migratable network devices back to the
7473          * initial network namespace
7474          */
7475         rtnl_lock();
7476         for_each_netdev_safe(net, dev, aux) {
7477                 int err;
7478                 char fb_name[IFNAMSIZ];
7479 
7480                 /* Ignore unmoveable devices (i.e. loopback) */
7481                 if (dev->features & NETIF_F_NETNS_LOCAL)
7482                         continue;
7483 
7484                 /* Leave virtual devices for the generic cleanup */
7485                 if (dev->rtnl_link_ops)
7486                         continue;
7487 
7488                 /* Push remaining network devices to init_net */
7489                 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
7490                 err = dev_change_net_namespace(dev, &init_net, fb_name);
7491                 if (err) {
7492                         pr_emerg("%s: failed to move %s to init_net: %d\n",
7493                                  __func__, dev->name, err);
7494                         BUG();
7495                 }
7496         }
7497         rtnl_unlock();
7498 }
7499 
7500 static void __net_exit rtnl_lock_unregistering(struct list_head *net_list)
7501 {
7502         /* Return with the rtnl_lock held when there are no network
7503          * devices unregistering in any network namespace in net_list.
7504          */
7505         struct net *net;
7506         bool unregistering;
7507         DEFINE_WAIT_FUNC(wait, woken_wake_function);
7508 
7509         add_wait_queue(&netdev_unregistering_wq, &wait);
7510         for (;;) {
7511                 unregistering = false;
7512                 rtnl_lock();
7513                 list_for_each_entry(net, net_list, exit_list) {
7514                         if (net->dev_unreg_count > 0) {
7515                                 unregistering = true;
7516                                 break;
7517                         }
7518                 }
7519                 if (!unregistering)
7520                         break;
7521                 __rtnl_unlock();
7522 
7523                 wait_woken(&wait, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
7524         }
7525         remove_wait_queue(&netdev_unregistering_wq, &wait);
7526 }
7527 
7528 static void __net_exit default_device_exit_batch(struct list_head *net_list)
7529 {
7530         /* At exit all network devices most be removed from a network
7531          * namespace.  Do this in the reverse order of registration.
7532          * Do this across as many network namespaces as possible to
7533          * improve batching efficiency.
7534          */
7535         struct net_device *dev;
7536         struct net *net;
7537         LIST_HEAD(dev_kill_list);
7538 
7539         /* To prevent network device cleanup code from dereferencing
7540          * loopback devices or network devices that have been freed
7541          * wait here for all pending unregistrations to complete,
7542          * before unregistring the loopback device and allowing the
7543          * network namespace be freed.
7544          *
7545          * The netdev todo list containing all network devices
7546          * unregistrations that happen in default_device_exit_batch
7547          * will run in the rtnl_unlock() at the end of
7548          * default_device_exit_batch.
7549          */
7550         rtnl_lock_unregistering(net_list);
7551         list_for_each_entry(net, net_list, exit_list) {
7552                 for_each_netdev_reverse(net, dev) {
7553                         if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink)
7554                                 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
7555                         else
7556                                 unregister_netdevice_queue(dev, &dev_kill_list);
7557                 }
7558         }
7559         unregister_netdevice_many(&dev_kill_list);
7560         rtnl_unlock();
7561 }
7562 
7563 static struct pernet_operations __net_initdata default_device_ops = {
7564         .exit = default_device_exit,
7565         .exit_batch = default_device_exit_batch,
7566 };
7567 
7568 /*
7569  *      Initialize the DEV module. At boot time this walks the device list and
7570  *      unhooks any devices that fail to initialise (normally hardware not
7571  *      present) and leaves us with a valid list of present and active devices.
7572  *
7573  */
7574 
7575 /*
7576  *       This is called single threaded during boot, so no need
7577  *       to take the rtnl semaphore.
7578  */
7579 static int __init net_dev_init(void)
7580 {
7581         int i, rc = -ENOMEM;
7582 
7583         BUG_ON(!dev_boot_phase);
7584 
7585         if (dev_proc_init())
7586                 goto out;
7587 
7588         if (netdev_kobject_init())
7589                 goto out;
7590 
7591         INIT_LIST_HEAD(&ptype_all);
7592         for (i = 0; i < PTYPE_HASH_SIZE; i++)
7593                 INIT_LIST_HEAD(&ptype_base[i]);
7594 
7595         INIT_LIST_HEAD(&offload_base);
7596 
7597         if (register_pernet_subsys(&netdev_net_ops))
7598                 goto out;
7599 
7600         /*
7601          *      Initialise the packet receive queues.
7602          */
7603 
7604         for_each_possible_cpu(i) {
7605                 struct softnet_data *sd = &per_cpu(softnet_data, i);
7606 
7607                 skb_queue_head_init(&sd->input_pkt_queue);
7608                 skb_queue_head_init(&sd->process_queue);
7609                 INIT_LIST_HEAD(&sd->poll_list);
7610                 sd->output_queue_tailp = &sd->output_queue;
7611 #ifdef CONFIG_RPS
7612                 sd->csd.func = rps_trigger_softirq;
7613                 sd->csd.info = sd;
7614                 sd->cpu = i;
7615 #endif
7616 
7617                 sd->backlog.poll = process_backlog;
7618                 sd->backlog.weight = weight_p;
7619         }
7620 
7621         dev_boot_phase = 0;
7622 
7623         /* The loopback device is special if any other network devices
7624          * is present in a network namespace the loopback device must
7625          * be present. Since we now dynamically allocate and free the
7626          * loopback device ensure this invariant is maintained by
7627          * keeping the loopback device as the first device on the
7628          * list of network devices.  Ensuring the loopback devices
7629          * is the first device that appears and the last network device
7630          * that disappears.
7631          */
7632         if (register_pernet_device(&loopback_net_ops))
7633                 goto out;
7634 
7635         if (register_pernet_device(&default_device_ops))
7636                 goto out;
7637 
7638         open_softirq(NET_TX_SOFTIRQ, net_tx_action);
7639         open_softirq(NET_RX_SOFTIRQ, net_rx_action);
7640 
7641         hotcpu_notifier(dev_cpu_callback, 0);
7642         dst_init();
7643         rc = 0;
7644 out:
7645         return rc;
7646 }
7647 
7648 subsys_initcall(net_dev_init);
7649 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us