Version:  2.0.40 2.2.26 2.4.37 2.6.39 3.0 3.1 3.2 3.3 3.4 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14 3.15

Linux/drivers/net/vmxnet3/vmxnet3_drv.c

  1 /*
  2  * Linux driver for VMware's vmxnet3 ethernet NIC.
  3  *
  4  * Copyright (C) 2008-2009, VMware, Inc. All Rights Reserved.
  5  *
  6  * This program is free software; you can redistribute it and/or modify it
  7  * under the terms of the GNU General Public License as published by the
  8  * Free Software Foundation; version 2 of the License and no later version.
  9  *
 10  * This program is distributed in the hope that it will be useful, but
 11  * WITHOUT ANY WARRANTY; without even the implied warranty of
 12  * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
 13  * NON INFRINGEMENT. See the GNU General Public License for more
 14  * details.
 15  *
 16  * You should have received a copy of the GNU General Public License
 17  * along with this program; if not, write to the Free Software
 18  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 19  *
 20  * The full GNU General Public License is included in this distribution in
 21  * the file called "COPYING".
 22  *
 23  * Maintained by: Shreyas Bhatewara <pv-drivers@vmware.com>
 24  *
 25  */
 26 
 27 #include <linux/module.h>
 28 #include <net/ip6_checksum.h>
 29 
 30 #include "vmxnet3_int.h"
 31 
 32 char vmxnet3_driver_name[] = "vmxnet3";
 33 #define VMXNET3_DRIVER_DESC "VMware vmxnet3 virtual NIC driver"
 34 
 35 /*
 36  * PCI Device ID Table
 37  * Last entry must be all 0s
 38  */
 39 static DEFINE_PCI_DEVICE_TABLE(vmxnet3_pciid_table) = {
 40         {PCI_VDEVICE(VMWARE, PCI_DEVICE_ID_VMWARE_VMXNET3)},
 41         {0}
 42 };
 43 
 44 MODULE_DEVICE_TABLE(pci, vmxnet3_pciid_table);
 45 
 46 static int enable_mq = 1;
 47 
 48 static void
 49 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac);
 50 
 51 /*
 52  *    Enable/Disable the given intr
 53  */
 54 static void
 55 vmxnet3_enable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx)
 56 {
 57         VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 0);
 58 }
 59 
 60 
 61 static void
 62 vmxnet3_disable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx)
 63 {
 64         VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 1);
 65 }
 66 
 67 
 68 /*
 69  *    Enable/Disable all intrs used by the device
 70  */
 71 static void
 72 vmxnet3_enable_all_intrs(struct vmxnet3_adapter *adapter)
 73 {
 74         int i;
 75 
 76         for (i = 0; i < adapter->intr.num_intrs; i++)
 77                 vmxnet3_enable_intr(adapter, i);
 78         adapter->shared->devRead.intrConf.intrCtrl &=
 79                                         cpu_to_le32(~VMXNET3_IC_DISABLE_ALL);
 80 }
 81 
 82 
 83 static void
 84 vmxnet3_disable_all_intrs(struct vmxnet3_adapter *adapter)
 85 {
 86         int i;
 87 
 88         adapter->shared->devRead.intrConf.intrCtrl |=
 89                                         cpu_to_le32(VMXNET3_IC_DISABLE_ALL);
 90         for (i = 0; i < adapter->intr.num_intrs; i++)
 91                 vmxnet3_disable_intr(adapter, i);
 92 }
 93 
 94 
 95 static void
 96 vmxnet3_ack_events(struct vmxnet3_adapter *adapter, u32 events)
 97 {
 98         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_ECR, events);
 99 }
100 
101 
102 static bool
103 vmxnet3_tq_stopped(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
104 {
105         return tq->stopped;
106 }
107 
108 
109 static void
110 vmxnet3_tq_start(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
111 {
112         tq->stopped = false;
113         netif_start_subqueue(adapter->netdev, tq - adapter->tx_queue);
114 }
115 
116 
117 static void
118 vmxnet3_tq_wake(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
119 {
120         tq->stopped = false;
121         netif_wake_subqueue(adapter->netdev, (tq - adapter->tx_queue));
122 }
123 
124 
125 static void
126 vmxnet3_tq_stop(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
127 {
128         tq->stopped = true;
129         tq->num_stop++;
130         netif_stop_subqueue(adapter->netdev, (tq - adapter->tx_queue));
131 }
132 
133 
134 /*
135  * Check the link state. This may start or stop the tx queue.
136  */
137 static void
138 vmxnet3_check_link(struct vmxnet3_adapter *adapter, bool affectTxQueue)
139 {
140         u32 ret;
141         int i;
142         unsigned long flags;
143 
144         spin_lock_irqsave(&adapter->cmd_lock, flags);
145         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK);
146         ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
147         spin_unlock_irqrestore(&adapter->cmd_lock, flags);
148 
149         adapter->link_speed = ret >> 16;
150         if (ret & 1) { /* Link is up. */
151                 netdev_info(adapter->netdev, "NIC Link is Up %d Mbps\n",
152                             adapter->link_speed);
153                 netif_carrier_on(adapter->netdev);
154 
155                 if (affectTxQueue) {
156                         for (i = 0; i < adapter->num_tx_queues; i++)
157                                 vmxnet3_tq_start(&adapter->tx_queue[i],
158                                                  adapter);
159                 }
160         } else {
161                 netdev_info(adapter->netdev, "NIC Link is Down\n");
162                 netif_carrier_off(adapter->netdev);
163 
164                 if (affectTxQueue) {
165                         for (i = 0; i < adapter->num_tx_queues; i++)
166                                 vmxnet3_tq_stop(&adapter->tx_queue[i], adapter);
167                 }
168         }
169 }
170 
171 static void
172 vmxnet3_process_events(struct vmxnet3_adapter *adapter)
173 {
174         int i;
175         unsigned long flags;
176         u32 events = le32_to_cpu(adapter->shared->ecr);
177         if (!events)
178                 return;
179 
180         vmxnet3_ack_events(adapter, events);
181 
182         /* Check if link state has changed */
183         if (events & VMXNET3_ECR_LINK)
184                 vmxnet3_check_link(adapter, true);
185 
186         /* Check if there is an error on xmit/recv queues */
187         if (events & (VMXNET3_ECR_TQERR | VMXNET3_ECR_RQERR)) {
188                 spin_lock_irqsave(&adapter->cmd_lock, flags);
189                 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
190                                        VMXNET3_CMD_GET_QUEUE_STATUS);
191                 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
192 
193                 for (i = 0; i < adapter->num_tx_queues; i++)
194                         if (adapter->tqd_start[i].status.stopped)
195                                 dev_err(&adapter->netdev->dev,
196                                         "%s: tq[%d] error 0x%x\n",
197                                         adapter->netdev->name, i, le32_to_cpu(
198                                         adapter->tqd_start[i].status.error));
199                 for (i = 0; i < adapter->num_rx_queues; i++)
200                         if (adapter->rqd_start[i].status.stopped)
201                                 dev_err(&adapter->netdev->dev,
202                                         "%s: rq[%d] error 0x%x\n",
203                                         adapter->netdev->name, i,
204                                         adapter->rqd_start[i].status.error);
205 
206                 schedule_work(&adapter->work);
207         }
208 }
209 
210 #ifdef __BIG_ENDIAN_BITFIELD
211 /*
212  * The device expects the bitfields in shared structures to be written in
213  * little endian. When CPU is big endian, the following routines are used to
214  * correctly read and write into ABI.
215  * The general technique used here is : double word bitfields are defined in
216  * opposite order for big endian architecture. Then before reading them in
217  * driver the complete double word is translated using le32_to_cpu. Similarly
218  * After the driver writes into bitfields, cpu_to_le32 is used to translate the
219  * double words into required format.
220  * In order to avoid touching bits in shared structure more than once, temporary
221  * descriptors are used. These are passed as srcDesc to following functions.
222  */
223 static void vmxnet3_RxDescToCPU(const struct Vmxnet3_RxDesc *srcDesc,
224                                 struct Vmxnet3_RxDesc *dstDesc)
225 {
226         u32 *src = (u32 *)srcDesc + 2;
227         u32 *dst = (u32 *)dstDesc + 2;
228         dstDesc->addr = le64_to_cpu(srcDesc->addr);
229         *dst = le32_to_cpu(*src);
230         dstDesc->ext1 = le32_to_cpu(srcDesc->ext1);
231 }
232 
233 static void vmxnet3_TxDescToLe(const struct Vmxnet3_TxDesc *srcDesc,
234                                struct Vmxnet3_TxDesc *dstDesc)
235 {
236         int i;
237         u32 *src = (u32 *)(srcDesc + 1);
238         u32 *dst = (u32 *)(dstDesc + 1);
239 
240         /* Working backwards so that the gen bit is set at the end. */
241         for (i = 2; i > 0; i--) {
242                 src--;
243                 dst--;
244                 *dst = cpu_to_le32(*src);
245         }
246 }
247 
248 
249 static void vmxnet3_RxCompToCPU(const struct Vmxnet3_RxCompDesc *srcDesc,
250                                 struct Vmxnet3_RxCompDesc *dstDesc)
251 {
252         int i = 0;
253         u32 *src = (u32 *)srcDesc;
254         u32 *dst = (u32 *)dstDesc;
255         for (i = 0; i < sizeof(struct Vmxnet3_RxCompDesc) / sizeof(u32); i++) {
256                 *dst = le32_to_cpu(*src);
257                 src++;
258                 dst++;
259         }
260 }
261 
262 
263 /* Used to read bitfield values from double words. */
264 static u32 get_bitfield32(const __le32 *bitfield, u32 pos, u32 size)
265 {
266         u32 temp = le32_to_cpu(*bitfield);
267         u32 mask = ((1 << size) - 1) << pos;
268         temp &= mask;
269         temp >>= pos;
270         return temp;
271 }
272 
273 
274 
275 #endif  /* __BIG_ENDIAN_BITFIELD */
276 
277 #ifdef __BIG_ENDIAN_BITFIELD
278 
279 #   define VMXNET3_TXDESC_GET_GEN(txdesc) get_bitfield32(((const __le32 *) \
280                         txdesc) + VMXNET3_TXD_GEN_DWORD_SHIFT, \
281                         VMXNET3_TXD_GEN_SHIFT, VMXNET3_TXD_GEN_SIZE)
282 #   define VMXNET3_TXDESC_GET_EOP(txdesc) get_bitfield32(((const __le32 *) \
283                         txdesc) + VMXNET3_TXD_EOP_DWORD_SHIFT, \
284                         VMXNET3_TXD_EOP_SHIFT, VMXNET3_TXD_EOP_SIZE)
285 #   define VMXNET3_TCD_GET_GEN(tcd) get_bitfield32(((const __le32 *)tcd) + \
286                         VMXNET3_TCD_GEN_DWORD_SHIFT, VMXNET3_TCD_GEN_SHIFT, \
287                         VMXNET3_TCD_GEN_SIZE)
288 #   define VMXNET3_TCD_GET_TXIDX(tcd) get_bitfield32((const __le32 *)tcd, \
289                         VMXNET3_TCD_TXIDX_SHIFT, VMXNET3_TCD_TXIDX_SIZE)
290 #   define vmxnet3_getRxComp(dstrcd, rcd, tmp) do { \
291                         (dstrcd) = (tmp); \
292                         vmxnet3_RxCompToCPU((rcd), (tmp)); \
293                 } while (0)
294 #   define vmxnet3_getRxDesc(dstrxd, rxd, tmp) do { \
295                         (dstrxd) = (tmp); \
296                         vmxnet3_RxDescToCPU((rxd), (tmp)); \
297                 } while (0)
298 
299 #else
300 
301 #   define VMXNET3_TXDESC_GET_GEN(txdesc) ((txdesc)->gen)
302 #   define VMXNET3_TXDESC_GET_EOP(txdesc) ((txdesc)->eop)
303 #   define VMXNET3_TCD_GET_GEN(tcd) ((tcd)->gen)
304 #   define VMXNET3_TCD_GET_TXIDX(tcd) ((tcd)->txdIdx)
305 #   define vmxnet3_getRxComp(dstrcd, rcd, tmp) (dstrcd) = (rcd)
306 #   define vmxnet3_getRxDesc(dstrxd, rxd, tmp) (dstrxd) = (rxd)
307 
308 #endif /* __BIG_ENDIAN_BITFIELD  */
309 
310 
311 static void
312 vmxnet3_unmap_tx_buf(struct vmxnet3_tx_buf_info *tbi,
313                      struct pci_dev *pdev)
314 {
315         if (tbi->map_type == VMXNET3_MAP_SINGLE)
316                 dma_unmap_single(&pdev->dev, tbi->dma_addr, tbi->len,
317                                  PCI_DMA_TODEVICE);
318         else if (tbi->map_type == VMXNET3_MAP_PAGE)
319                 dma_unmap_page(&pdev->dev, tbi->dma_addr, tbi->len,
320                                PCI_DMA_TODEVICE);
321         else
322                 BUG_ON(tbi->map_type != VMXNET3_MAP_NONE);
323 
324         tbi->map_type = VMXNET3_MAP_NONE; /* to help debugging */
325 }
326 
327 
328 static int
329 vmxnet3_unmap_pkt(u32 eop_idx, struct vmxnet3_tx_queue *tq,
330                   struct pci_dev *pdev, struct vmxnet3_adapter *adapter)
331 {
332         struct sk_buff *skb;
333         int entries = 0;
334 
335         /* no out of order completion */
336         BUG_ON(tq->buf_info[eop_idx].sop_idx != tq->tx_ring.next2comp);
337         BUG_ON(VMXNET3_TXDESC_GET_EOP(&(tq->tx_ring.base[eop_idx].txd)) != 1);
338 
339         skb = tq->buf_info[eop_idx].skb;
340         BUG_ON(skb == NULL);
341         tq->buf_info[eop_idx].skb = NULL;
342 
343         VMXNET3_INC_RING_IDX_ONLY(eop_idx, tq->tx_ring.size);
344 
345         while (tq->tx_ring.next2comp != eop_idx) {
346                 vmxnet3_unmap_tx_buf(tq->buf_info + tq->tx_ring.next2comp,
347                                      pdev);
348 
349                 /* update next2comp w/o tx_lock. Since we are marking more,
350                  * instead of less, tx ring entries avail, the worst case is
351                  * that the tx routine incorrectly re-queues a pkt due to
352                  * insufficient tx ring entries.
353                  */
354                 vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring);
355                 entries++;
356         }
357 
358         dev_kfree_skb_any(skb);
359         return entries;
360 }
361 
362 
363 static int
364 vmxnet3_tq_tx_complete(struct vmxnet3_tx_queue *tq,
365                         struct vmxnet3_adapter *adapter)
366 {
367         int completed = 0;
368         union Vmxnet3_GenericDesc *gdesc;
369 
370         gdesc = tq->comp_ring.base + tq->comp_ring.next2proc;
371         while (VMXNET3_TCD_GET_GEN(&gdesc->tcd) == tq->comp_ring.gen) {
372                 completed += vmxnet3_unmap_pkt(VMXNET3_TCD_GET_TXIDX(
373                                                &gdesc->tcd), tq, adapter->pdev,
374                                                adapter);
375 
376                 vmxnet3_comp_ring_adv_next2proc(&tq->comp_ring);
377                 gdesc = tq->comp_ring.base + tq->comp_ring.next2proc;
378         }
379 
380         if (completed) {
381                 spin_lock(&tq->tx_lock);
382                 if (unlikely(vmxnet3_tq_stopped(tq, adapter) &&
383                              vmxnet3_cmd_ring_desc_avail(&tq->tx_ring) >
384                              VMXNET3_WAKE_QUEUE_THRESHOLD(tq) &&
385                              netif_carrier_ok(adapter->netdev))) {
386                         vmxnet3_tq_wake(tq, adapter);
387                 }
388                 spin_unlock(&tq->tx_lock);
389         }
390         return completed;
391 }
392 
393 
394 static void
395 vmxnet3_tq_cleanup(struct vmxnet3_tx_queue *tq,
396                    struct vmxnet3_adapter *adapter)
397 {
398         int i;
399 
400         while (tq->tx_ring.next2comp != tq->tx_ring.next2fill) {
401                 struct vmxnet3_tx_buf_info *tbi;
402 
403                 tbi = tq->buf_info + tq->tx_ring.next2comp;
404 
405                 vmxnet3_unmap_tx_buf(tbi, adapter->pdev);
406                 if (tbi->skb) {
407                         dev_kfree_skb_any(tbi->skb);
408                         tbi->skb = NULL;
409                 }
410                 vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring);
411         }
412 
413         /* sanity check, verify all buffers are indeed unmapped and freed */
414         for (i = 0; i < tq->tx_ring.size; i++) {
415                 BUG_ON(tq->buf_info[i].skb != NULL ||
416                        tq->buf_info[i].map_type != VMXNET3_MAP_NONE);
417         }
418 
419         tq->tx_ring.gen = VMXNET3_INIT_GEN;
420         tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0;
421 
422         tq->comp_ring.gen = VMXNET3_INIT_GEN;
423         tq->comp_ring.next2proc = 0;
424 }
425 
426 
427 static void
428 vmxnet3_tq_destroy(struct vmxnet3_tx_queue *tq,
429                    struct vmxnet3_adapter *adapter)
430 {
431         if (tq->tx_ring.base) {
432                 dma_free_coherent(&adapter->pdev->dev, tq->tx_ring.size *
433                                   sizeof(struct Vmxnet3_TxDesc),
434                                   tq->tx_ring.base, tq->tx_ring.basePA);
435                 tq->tx_ring.base = NULL;
436         }
437         if (tq->data_ring.base) {
438                 dma_free_coherent(&adapter->pdev->dev, tq->data_ring.size *
439                                   sizeof(struct Vmxnet3_TxDataDesc),
440                                   tq->data_ring.base, tq->data_ring.basePA);
441                 tq->data_ring.base = NULL;
442         }
443         if (tq->comp_ring.base) {
444                 dma_free_coherent(&adapter->pdev->dev, tq->comp_ring.size *
445                                   sizeof(struct Vmxnet3_TxCompDesc),
446                                   tq->comp_ring.base, tq->comp_ring.basePA);
447                 tq->comp_ring.base = NULL;
448         }
449         if (tq->buf_info) {
450                 dma_free_coherent(&adapter->pdev->dev,
451                                   tq->tx_ring.size * sizeof(tq->buf_info[0]),
452                                   tq->buf_info, tq->buf_info_pa);
453                 tq->buf_info = NULL;
454         }
455 }
456 
457 
458 /* Destroy all tx queues */
459 void
460 vmxnet3_tq_destroy_all(struct vmxnet3_adapter *adapter)
461 {
462         int i;
463 
464         for (i = 0; i < adapter->num_tx_queues; i++)
465                 vmxnet3_tq_destroy(&adapter->tx_queue[i], adapter);
466 }
467 
468 
469 static void
470 vmxnet3_tq_init(struct vmxnet3_tx_queue *tq,
471                 struct vmxnet3_adapter *adapter)
472 {
473         int i;
474 
475         /* reset the tx ring contents to 0 and reset the tx ring states */
476         memset(tq->tx_ring.base, 0, tq->tx_ring.size *
477                sizeof(struct Vmxnet3_TxDesc));
478         tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0;
479         tq->tx_ring.gen = VMXNET3_INIT_GEN;
480 
481         memset(tq->data_ring.base, 0, tq->data_ring.size *
482                sizeof(struct Vmxnet3_TxDataDesc));
483 
484         /* reset the tx comp ring contents to 0 and reset comp ring states */
485         memset(tq->comp_ring.base, 0, tq->comp_ring.size *
486                sizeof(struct Vmxnet3_TxCompDesc));
487         tq->comp_ring.next2proc = 0;
488         tq->comp_ring.gen = VMXNET3_INIT_GEN;
489 
490         /* reset the bookkeeping data */
491         memset(tq->buf_info, 0, sizeof(tq->buf_info[0]) * tq->tx_ring.size);
492         for (i = 0; i < tq->tx_ring.size; i++)
493                 tq->buf_info[i].map_type = VMXNET3_MAP_NONE;
494 
495         /* stats are not reset */
496 }
497 
498 
499 static int
500 vmxnet3_tq_create(struct vmxnet3_tx_queue *tq,
501                   struct vmxnet3_adapter *adapter)
502 {
503         size_t sz;
504 
505         BUG_ON(tq->tx_ring.base || tq->data_ring.base ||
506                tq->comp_ring.base || tq->buf_info);
507 
508         tq->tx_ring.base = dma_alloc_coherent(&adapter->pdev->dev,
509                         tq->tx_ring.size * sizeof(struct Vmxnet3_TxDesc),
510                         &tq->tx_ring.basePA, GFP_KERNEL);
511         if (!tq->tx_ring.base) {
512                 netdev_err(adapter->netdev, "failed to allocate tx ring\n");
513                 goto err;
514         }
515 
516         tq->data_ring.base = dma_alloc_coherent(&adapter->pdev->dev,
517                         tq->data_ring.size * sizeof(struct Vmxnet3_TxDataDesc),
518                         &tq->data_ring.basePA, GFP_KERNEL);
519         if (!tq->data_ring.base) {
520                 netdev_err(adapter->netdev, "failed to allocate data ring\n");
521                 goto err;
522         }
523 
524         tq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev,
525                         tq->comp_ring.size * sizeof(struct Vmxnet3_TxCompDesc),
526                         &tq->comp_ring.basePA, GFP_KERNEL);
527         if (!tq->comp_ring.base) {
528                 netdev_err(adapter->netdev, "failed to allocate tx comp ring\n");
529                 goto err;
530         }
531 
532         sz = tq->tx_ring.size * sizeof(tq->buf_info[0]);
533         tq->buf_info = dma_zalloc_coherent(&adapter->pdev->dev, sz,
534                                            &tq->buf_info_pa, GFP_KERNEL);
535         if (!tq->buf_info)
536                 goto err;
537 
538         return 0;
539 
540 err:
541         vmxnet3_tq_destroy(tq, adapter);
542         return -ENOMEM;
543 }
544 
545 static void
546 vmxnet3_tq_cleanup_all(struct vmxnet3_adapter *adapter)
547 {
548         int i;
549 
550         for (i = 0; i < adapter->num_tx_queues; i++)
551                 vmxnet3_tq_cleanup(&adapter->tx_queue[i], adapter);
552 }
553 
554 /*
555  *    starting from ring->next2fill, allocate rx buffers for the given ring
556  *    of the rx queue and update the rx desc. stop after @num_to_alloc buffers
557  *    are allocated or allocation fails
558  */
559 
560 static int
561 vmxnet3_rq_alloc_rx_buf(struct vmxnet3_rx_queue *rq, u32 ring_idx,
562                         int num_to_alloc, struct vmxnet3_adapter *adapter)
563 {
564         int num_allocated = 0;
565         struct vmxnet3_rx_buf_info *rbi_base = rq->buf_info[ring_idx];
566         struct vmxnet3_cmd_ring *ring = &rq->rx_ring[ring_idx];
567         u32 val;
568 
569         while (num_allocated <= num_to_alloc) {
570                 struct vmxnet3_rx_buf_info *rbi;
571                 union Vmxnet3_GenericDesc *gd;
572 
573                 rbi = rbi_base + ring->next2fill;
574                 gd = ring->base + ring->next2fill;
575 
576                 if (rbi->buf_type == VMXNET3_RX_BUF_SKB) {
577                         if (rbi->skb == NULL) {
578                                 rbi->skb = __netdev_alloc_skb_ip_align(adapter->netdev,
579                                                                        rbi->len,
580                                                                        GFP_KERNEL);
581                                 if (unlikely(rbi->skb == NULL)) {
582                                         rq->stats.rx_buf_alloc_failure++;
583                                         break;
584                                 }
585 
586                                 rbi->dma_addr = dma_map_single(
587                                                 &adapter->pdev->dev,
588                                                 rbi->skb->data, rbi->len,
589                                                 PCI_DMA_FROMDEVICE);
590                         } else {
591                                 /* rx buffer skipped by the device */
592                         }
593                         val = VMXNET3_RXD_BTYPE_HEAD << VMXNET3_RXD_BTYPE_SHIFT;
594                 } else {
595                         BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE ||
596                                rbi->len  != PAGE_SIZE);
597 
598                         if (rbi->page == NULL) {
599                                 rbi->page = alloc_page(GFP_ATOMIC);
600                                 if (unlikely(rbi->page == NULL)) {
601                                         rq->stats.rx_buf_alloc_failure++;
602                                         break;
603                                 }
604                                 rbi->dma_addr = dma_map_page(
605                                                 &adapter->pdev->dev,
606                                                 rbi->page, 0, PAGE_SIZE,
607                                                 PCI_DMA_FROMDEVICE);
608                         } else {
609                                 /* rx buffers skipped by the device */
610                         }
611                         val = VMXNET3_RXD_BTYPE_BODY << VMXNET3_RXD_BTYPE_SHIFT;
612                 }
613 
614                 BUG_ON(rbi->dma_addr == 0);
615                 gd->rxd.addr = cpu_to_le64(rbi->dma_addr);
616                 gd->dword[2] = cpu_to_le32((!ring->gen << VMXNET3_RXD_GEN_SHIFT)
617                                            | val | rbi->len);
618 
619                 /* Fill the last buffer but dont mark it ready, or else the
620                  * device will think that the queue is full */
621                 if (num_allocated == num_to_alloc)
622                         break;
623 
624                 gd->dword[2] |= cpu_to_le32(ring->gen << VMXNET3_RXD_GEN_SHIFT);
625                 num_allocated++;
626                 vmxnet3_cmd_ring_adv_next2fill(ring);
627         }
628 
629         netdev_dbg(adapter->netdev,
630                 "alloc_rx_buf: %d allocated, next2fill %u, next2comp %u\n",
631                 num_allocated, ring->next2fill, ring->next2comp);
632 
633         /* so that the device can distinguish a full ring and an empty ring */
634         BUG_ON(num_allocated != 0 && ring->next2fill == ring->next2comp);
635 
636         return num_allocated;
637 }
638 
639 
640 static void
641 vmxnet3_append_frag(struct sk_buff *skb, struct Vmxnet3_RxCompDesc *rcd,
642                     struct vmxnet3_rx_buf_info *rbi)
643 {
644         struct skb_frag_struct *frag = skb_shinfo(skb)->frags +
645                 skb_shinfo(skb)->nr_frags;
646 
647         BUG_ON(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS);
648 
649         __skb_frag_set_page(frag, rbi->page);
650         frag->page_offset = 0;
651         skb_frag_size_set(frag, rcd->len);
652         skb->data_len += rcd->len;
653         skb->truesize += PAGE_SIZE;
654         skb_shinfo(skb)->nr_frags++;
655 }
656 
657 
658 static void
659 vmxnet3_map_pkt(struct sk_buff *skb, struct vmxnet3_tx_ctx *ctx,
660                 struct vmxnet3_tx_queue *tq, struct pci_dev *pdev,
661                 struct vmxnet3_adapter *adapter)
662 {
663         u32 dw2, len;
664         unsigned long buf_offset;
665         int i;
666         union Vmxnet3_GenericDesc *gdesc;
667         struct vmxnet3_tx_buf_info *tbi = NULL;
668 
669         BUG_ON(ctx->copy_size > skb_headlen(skb));
670 
671         /* use the previous gen bit for the SOP desc */
672         dw2 = (tq->tx_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT;
673 
674         ctx->sop_txd = tq->tx_ring.base + tq->tx_ring.next2fill;
675         gdesc = ctx->sop_txd; /* both loops below can be skipped */
676 
677         /* no need to map the buffer if headers are copied */
678         if (ctx->copy_size) {
679                 ctx->sop_txd->txd.addr = cpu_to_le64(tq->data_ring.basePA +
680                                         tq->tx_ring.next2fill *
681                                         sizeof(struct Vmxnet3_TxDataDesc));
682                 ctx->sop_txd->dword[2] = cpu_to_le32(dw2 | ctx->copy_size);
683                 ctx->sop_txd->dword[3] = 0;
684 
685                 tbi = tq->buf_info + tq->tx_ring.next2fill;
686                 tbi->map_type = VMXNET3_MAP_NONE;
687 
688                 netdev_dbg(adapter->netdev,
689                         "txd[%u]: 0x%Lx 0x%x 0x%x\n",
690                         tq->tx_ring.next2fill,
691                         le64_to_cpu(ctx->sop_txd->txd.addr),
692                         ctx->sop_txd->dword[2], ctx->sop_txd->dword[3]);
693                 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
694 
695                 /* use the right gen for non-SOP desc */
696                 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
697         }
698 
699         /* linear part can use multiple tx desc if it's big */
700         len = skb_headlen(skb) - ctx->copy_size;
701         buf_offset = ctx->copy_size;
702         while (len) {
703                 u32 buf_size;
704 
705                 if (len < VMXNET3_MAX_TX_BUF_SIZE) {
706                         buf_size = len;
707                         dw2 |= len;
708                 } else {
709                         buf_size = VMXNET3_MAX_TX_BUF_SIZE;
710                         /* spec says that for TxDesc.len, 0 == 2^14 */
711                 }
712 
713                 tbi = tq->buf_info + tq->tx_ring.next2fill;
714                 tbi->map_type = VMXNET3_MAP_SINGLE;
715                 tbi->dma_addr = dma_map_single(&adapter->pdev->dev,
716                                 skb->data + buf_offset, buf_size,
717                                 PCI_DMA_TODEVICE);
718 
719                 tbi->len = buf_size;
720 
721                 gdesc = tq->tx_ring.base + tq->tx_ring.next2fill;
722                 BUG_ON(gdesc->txd.gen == tq->tx_ring.gen);
723 
724                 gdesc->txd.addr = cpu_to_le64(tbi->dma_addr);
725                 gdesc->dword[2] = cpu_to_le32(dw2);
726                 gdesc->dword[3] = 0;
727 
728                 netdev_dbg(adapter->netdev,
729                         "txd[%u]: 0x%Lx 0x%x 0x%x\n",
730                         tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr),
731                         le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]);
732                 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
733                 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
734 
735                 len -= buf_size;
736                 buf_offset += buf_size;
737         }
738 
739         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
740                 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
741                 u32 buf_size;
742 
743                 buf_offset = 0;
744                 len = skb_frag_size(frag);
745                 while (len) {
746                         tbi = tq->buf_info + tq->tx_ring.next2fill;
747                         if (len < VMXNET3_MAX_TX_BUF_SIZE) {
748                                 buf_size = len;
749                                 dw2 |= len;
750                         } else {
751                                 buf_size = VMXNET3_MAX_TX_BUF_SIZE;
752                                 /* spec says that for TxDesc.len, 0 == 2^14 */
753                         }
754                         tbi->map_type = VMXNET3_MAP_PAGE;
755                         tbi->dma_addr = skb_frag_dma_map(&adapter->pdev->dev, frag,
756                                                          buf_offset, buf_size,
757                                                          DMA_TO_DEVICE);
758 
759                         tbi->len = buf_size;
760 
761                         gdesc = tq->tx_ring.base + tq->tx_ring.next2fill;
762                         BUG_ON(gdesc->txd.gen == tq->tx_ring.gen);
763 
764                         gdesc->txd.addr = cpu_to_le64(tbi->dma_addr);
765                         gdesc->dword[2] = cpu_to_le32(dw2);
766                         gdesc->dword[3] = 0;
767 
768                         netdev_dbg(adapter->netdev,
769                                 "txd[%u]: 0x%llu %u %u\n",
770                                 tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr),
771                                 le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]);
772                         vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
773                         dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
774 
775                         len -= buf_size;
776                         buf_offset += buf_size;
777                 }
778         }
779 
780         ctx->eop_txd = gdesc;
781 
782         /* set the last buf_info for the pkt */
783         tbi->skb = skb;
784         tbi->sop_idx = ctx->sop_txd - tq->tx_ring.base;
785 }
786 
787 
788 /* Init all tx queues */
789 static void
790 vmxnet3_tq_init_all(struct vmxnet3_adapter *adapter)
791 {
792         int i;
793 
794         for (i = 0; i < adapter->num_tx_queues; i++)
795                 vmxnet3_tq_init(&adapter->tx_queue[i], adapter);
796 }
797 
798 
799 /*
800  *    parse and copy relevant protocol headers:
801  *      For a tso pkt, relevant headers are L2/3/4 including options
802  *      For a pkt requesting csum offloading, they are L2/3 and may include L4
803  *      if it's a TCP/UDP pkt
804  *
805  * Returns:
806  *    -1:  error happens during parsing
807  *     0:  protocol headers parsed, but too big to be copied
808  *     1:  protocol headers parsed and copied
809  *
810  * Other effects:
811  *    1. related *ctx fields are updated.
812  *    2. ctx->copy_size is # of bytes copied
813  *    3. the portion copied is guaranteed to be in the linear part
814  *
815  */
816 static int
817 vmxnet3_parse_and_copy_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
818                            struct vmxnet3_tx_ctx *ctx,
819                            struct vmxnet3_adapter *adapter)
820 {
821         struct Vmxnet3_TxDataDesc *tdd;
822 
823         if (ctx->mss) { /* TSO */
824                 ctx->eth_ip_hdr_size = skb_transport_offset(skb);
825                 ctx->l4_hdr_size = tcp_hdrlen(skb);
826                 ctx->copy_size = ctx->eth_ip_hdr_size + ctx->l4_hdr_size;
827         } else {
828                 if (skb->ip_summed == CHECKSUM_PARTIAL) {
829                         ctx->eth_ip_hdr_size = skb_checksum_start_offset(skb);
830 
831                         if (ctx->ipv4) {
832                                 const struct iphdr *iph = ip_hdr(skb);
833 
834                                 if (iph->protocol == IPPROTO_TCP)
835                                         ctx->l4_hdr_size = tcp_hdrlen(skb);
836                                 else if (iph->protocol == IPPROTO_UDP)
837                                         ctx->l4_hdr_size = sizeof(struct udphdr);
838                                 else
839                                         ctx->l4_hdr_size = 0;
840                         } else {
841                                 /* for simplicity, don't copy L4 headers */
842                                 ctx->l4_hdr_size = 0;
843                         }
844                         ctx->copy_size = min(ctx->eth_ip_hdr_size +
845                                          ctx->l4_hdr_size, skb->len);
846                 } else {
847                         ctx->eth_ip_hdr_size = 0;
848                         ctx->l4_hdr_size = 0;
849                         /* copy as much as allowed */
850                         ctx->copy_size = min((unsigned int)VMXNET3_HDR_COPY_SIZE
851                                              , skb_headlen(skb));
852                 }
853 
854                 /* make sure headers are accessible directly */
855                 if (unlikely(!pskb_may_pull(skb, ctx->copy_size)))
856                         goto err;
857         }
858 
859         if (unlikely(ctx->copy_size > VMXNET3_HDR_COPY_SIZE)) {
860                 tq->stats.oversized_hdr++;
861                 ctx->copy_size = 0;
862                 return 0;
863         }
864 
865         tdd = tq->data_ring.base + tq->tx_ring.next2fill;
866 
867         memcpy(tdd->data, skb->data, ctx->copy_size);
868         netdev_dbg(adapter->netdev,
869                 "copy %u bytes to dataRing[%u]\n",
870                 ctx->copy_size, tq->tx_ring.next2fill);
871         return 1;
872 
873 err:
874         return -1;
875 }
876 
877 
878 static void
879 vmxnet3_prepare_tso(struct sk_buff *skb,
880                     struct vmxnet3_tx_ctx *ctx)
881 {
882         struct tcphdr *tcph = tcp_hdr(skb);
883 
884         if (ctx->ipv4) {
885                 struct iphdr *iph = ip_hdr(skb);
886 
887                 iph->check = 0;
888                 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0,
889                                                  IPPROTO_TCP, 0);
890         } else {
891                 struct ipv6hdr *iph = ipv6_hdr(skb);
892 
893                 tcph->check = ~csum_ipv6_magic(&iph->saddr, &iph->daddr, 0,
894                                                IPPROTO_TCP, 0);
895         }
896 }
897 
898 static int txd_estimate(const struct sk_buff *skb)
899 {
900         int count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1;
901         int i;
902 
903         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
904                 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
905 
906                 count += VMXNET3_TXD_NEEDED(skb_frag_size(frag));
907         }
908         return count;
909 }
910 
911 /*
912  * Transmits a pkt thru a given tq
913  * Returns:
914  *    NETDEV_TX_OK:      descriptors are setup successfully
915  *    NETDEV_TX_OK:      error occurred, the pkt is dropped
916  *    NETDEV_TX_BUSY:    tx ring is full, queue is stopped
917  *
918  * Side-effects:
919  *    1. tx ring may be changed
920  *    2. tq stats may be updated accordingly
921  *    3. shared->txNumDeferred may be updated
922  */
923 
924 static int
925 vmxnet3_tq_xmit(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
926                 struct vmxnet3_adapter *adapter, struct net_device *netdev)
927 {
928         int ret;
929         u32 count;
930         unsigned long flags;
931         struct vmxnet3_tx_ctx ctx;
932         union Vmxnet3_GenericDesc *gdesc;
933 #ifdef __BIG_ENDIAN_BITFIELD
934         /* Use temporary descriptor to avoid touching bits multiple times */
935         union Vmxnet3_GenericDesc tempTxDesc;
936 #endif
937 
938         count = txd_estimate(skb);
939 
940         ctx.ipv4 = (vlan_get_protocol(skb) == cpu_to_be16(ETH_P_IP));
941 
942         ctx.mss = skb_shinfo(skb)->gso_size;
943         if (ctx.mss) {
944                 if (skb_header_cloned(skb)) {
945                         if (unlikely(pskb_expand_head(skb, 0, 0,
946                                                       GFP_ATOMIC) != 0)) {
947                                 tq->stats.drop_tso++;
948                                 goto drop_pkt;
949                         }
950                         tq->stats.copy_skb_header++;
951                 }
952                 vmxnet3_prepare_tso(skb, &ctx);
953         } else {
954                 if (unlikely(count > VMXNET3_MAX_TXD_PER_PKT)) {
955 
956                         /* non-tso pkts must not use more than
957                          * VMXNET3_MAX_TXD_PER_PKT entries
958                          */
959                         if (skb_linearize(skb) != 0) {
960                                 tq->stats.drop_too_many_frags++;
961                                 goto drop_pkt;
962                         }
963                         tq->stats.linearized++;
964 
965                         /* recalculate the # of descriptors to use */
966                         count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1;
967                 }
968         }
969 
970         spin_lock_irqsave(&tq->tx_lock, flags);
971 
972         if (count > vmxnet3_cmd_ring_desc_avail(&tq->tx_ring)) {
973                 tq->stats.tx_ring_full++;
974                 netdev_dbg(adapter->netdev,
975                         "tx queue stopped on %s, next2comp %u"
976                         " next2fill %u\n", adapter->netdev->name,
977                         tq->tx_ring.next2comp, tq->tx_ring.next2fill);
978 
979                 vmxnet3_tq_stop(tq, adapter);
980                 spin_unlock_irqrestore(&tq->tx_lock, flags);
981                 return NETDEV_TX_BUSY;
982         }
983 
984 
985         ret = vmxnet3_parse_and_copy_hdr(skb, tq, &ctx, adapter);
986         if (ret >= 0) {
987                 BUG_ON(ret <= 0 && ctx.copy_size != 0);
988                 /* hdrs parsed, check against other limits */
989                 if (ctx.mss) {
990                         if (unlikely(ctx.eth_ip_hdr_size + ctx.l4_hdr_size >
991                                      VMXNET3_MAX_TX_BUF_SIZE)) {
992                                 goto hdr_too_big;
993                         }
994                 } else {
995                         if (skb->ip_summed == CHECKSUM_PARTIAL) {
996                                 if (unlikely(ctx.eth_ip_hdr_size +
997                                              skb->csum_offset >
998                                              VMXNET3_MAX_CSUM_OFFSET)) {
999                                         goto hdr_too_big;
1000                                 }
1001                         }
1002                 }
1003         } else {
1004                 tq->stats.drop_hdr_inspect_err++;
1005                 goto unlock_drop_pkt;
1006         }
1007 
1008         /* fill tx descs related to addr & len */
1009         vmxnet3_map_pkt(skb, &ctx, tq, adapter->pdev, adapter);
1010 
1011         /* setup the EOP desc */
1012         ctx.eop_txd->dword[3] = cpu_to_le32(VMXNET3_TXD_CQ | VMXNET3_TXD_EOP);
1013 
1014         /* setup the SOP desc */
1015 #ifdef __BIG_ENDIAN_BITFIELD
1016         gdesc = &tempTxDesc;
1017         gdesc->dword[2] = ctx.sop_txd->dword[2];
1018         gdesc->dword[3] = ctx.sop_txd->dword[3];
1019 #else
1020         gdesc = ctx.sop_txd;
1021 #endif
1022         if (ctx.mss) {
1023                 gdesc->txd.hlen = ctx.eth_ip_hdr_size + ctx.l4_hdr_size;
1024                 gdesc->txd.om = VMXNET3_OM_TSO;
1025                 gdesc->txd.msscof = ctx.mss;
1026                 le32_add_cpu(&tq->shared->txNumDeferred, (skb->len -
1027                              gdesc->txd.hlen + ctx.mss - 1) / ctx.mss);
1028         } else {
1029                 if (skb->ip_summed == CHECKSUM_PARTIAL) {
1030                         gdesc->txd.hlen = ctx.eth_ip_hdr_size;
1031                         gdesc->txd.om = VMXNET3_OM_CSUM;
1032                         gdesc->txd.msscof = ctx.eth_ip_hdr_size +
1033                                             skb->csum_offset;
1034                 } else {
1035                         gdesc->txd.om = 0;
1036                         gdesc->txd.msscof = 0;
1037                 }
1038                 le32_add_cpu(&tq->shared->txNumDeferred, 1);
1039         }
1040 
1041         if (vlan_tx_tag_present(skb)) {
1042                 gdesc->txd.ti = 1;
1043                 gdesc->txd.tci = vlan_tx_tag_get(skb);
1044         }
1045 
1046         /* finally flips the GEN bit of the SOP desc. */
1047         gdesc->dword[2] = cpu_to_le32(le32_to_cpu(gdesc->dword[2]) ^
1048                                                   VMXNET3_TXD_GEN);
1049 #ifdef __BIG_ENDIAN_BITFIELD
1050         /* Finished updating in bitfields of Tx Desc, so write them in original
1051          * place.
1052          */
1053         vmxnet3_TxDescToLe((struct Vmxnet3_TxDesc *)gdesc,
1054                            (struct Vmxnet3_TxDesc *)ctx.sop_txd);
1055         gdesc = ctx.sop_txd;
1056 #endif
1057         netdev_dbg(adapter->netdev,
1058                 "txd[%u]: SOP 0x%Lx 0x%x 0x%x\n",
1059                 (u32)(ctx.sop_txd -
1060                 tq->tx_ring.base), le64_to_cpu(gdesc->txd.addr),
1061                 le32_to_cpu(gdesc->dword[2]), le32_to_cpu(gdesc->dword[3]));
1062 
1063         spin_unlock_irqrestore(&tq->tx_lock, flags);
1064 
1065         if (le32_to_cpu(tq->shared->txNumDeferred) >=
1066                                         le32_to_cpu(tq->shared->txThreshold)) {
1067                 tq->shared->txNumDeferred = 0;
1068                 VMXNET3_WRITE_BAR0_REG(adapter,
1069                                        VMXNET3_REG_TXPROD + tq->qid * 8,
1070                                        tq->tx_ring.next2fill);
1071         }
1072 
1073         return NETDEV_TX_OK;
1074 
1075 hdr_too_big:
1076         tq->stats.drop_oversized_hdr++;
1077 unlock_drop_pkt:
1078         spin_unlock_irqrestore(&tq->tx_lock, flags);
1079 drop_pkt:
1080         tq->stats.drop_total++;
1081         dev_kfree_skb_any(skb);
1082         return NETDEV_TX_OK;
1083 }
1084 
1085 
1086 static netdev_tx_t
1087 vmxnet3_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
1088 {
1089         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1090 
1091         BUG_ON(skb->queue_mapping > adapter->num_tx_queues);
1092         return vmxnet3_tq_xmit(skb,
1093                                &adapter->tx_queue[skb->queue_mapping],
1094                                adapter, netdev);
1095 }
1096 
1097 
1098 static void
1099 vmxnet3_rx_csum(struct vmxnet3_adapter *adapter,
1100                 struct sk_buff *skb,
1101                 union Vmxnet3_GenericDesc *gdesc)
1102 {
1103         if (!gdesc->rcd.cnc && adapter->netdev->features & NETIF_F_RXCSUM) {
1104                 /* typical case: TCP/UDP over IP and both csums are correct */
1105                 if ((le32_to_cpu(gdesc->dword[3]) & VMXNET3_RCD_CSUM_OK) ==
1106                                                         VMXNET3_RCD_CSUM_OK) {
1107                         skb->ip_summed = CHECKSUM_UNNECESSARY;
1108                         BUG_ON(!(gdesc->rcd.tcp || gdesc->rcd.udp));
1109                         BUG_ON(!(gdesc->rcd.v4  || gdesc->rcd.v6));
1110                         BUG_ON(gdesc->rcd.frg);
1111                 } else {
1112                         if (gdesc->rcd.csum) {
1113                                 skb->csum = htons(gdesc->rcd.csum);
1114                                 skb->ip_summed = CHECKSUM_PARTIAL;
1115                         } else {
1116                                 skb_checksum_none_assert(skb);
1117                         }
1118                 }
1119         } else {
1120                 skb_checksum_none_assert(skb);
1121         }
1122 }
1123 
1124 
1125 static void
1126 vmxnet3_rx_error(struct vmxnet3_rx_queue *rq, struct Vmxnet3_RxCompDesc *rcd,
1127                  struct vmxnet3_rx_ctx *ctx,  struct vmxnet3_adapter *adapter)
1128 {
1129         rq->stats.drop_err++;
1130         if (!rcd->fcs)
1131                 rq->stats.drop_fcs++;
1132 
1133         rq->stats.drop_total++;
1134 
1135         /*
1136          * We do not unmap and chain the rx buffer to the skb.
1137          * We basically pretend this buffer is not used and will be recycled
1138          * by vmxnet3_rq_alloc_rx_buf()
1139          */
1140 
1141         /*
1142          * ctx->skb may be NULL if this is the first and the only one
1143          * desc for the pkt
1144          */
1145         if (ctx->skb)
1146                 dev_kfree_skb_irq(ctx->skb);
1147 
1148         ctx->skb = NULL;
1149 }
1150 
1151 
1152 static int
1153 vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq,
1154                        struct vmxnet3_adapter *adapter, int quota)
1155 {
1156         static const u32 rxprod_reg[2] = {
1157                 VMXNET3_REG_RXPROD, VMXNET3_REG_RXPROD2
1158         };
1159         u32 num_rxd = 0;
1160         bool skip_page_frags = false;
1161         struct Vmxnet3_RxCompDesc *rcd;
1162         struct vmxnet3_rx_ctx *ctx = &rq->rx_ctx;
1163 #ifdef __BIG_ENDIAN_BITFIELD
1164         struct Vmxnet3_RxDesc rxCmdDesc;
1165         struct Vmxnet3_RxCompDesc rxComp;
1166 #endif
1167         vmxnet3_getRxComp(rcd, &rq->comp_ring.base[rq->comp_ring.next2proc].rcd,
1168                           &rxComp);
1169         while (rcd->gen == rq->comp_ring.gen) {
1170                 struct vmxnet3_rx_buf_info *rbi;
1171                 struct sk_buff *skb, *new_skb = NULL;
1172                 struct page *new_page = NULL;
1173                 int num_to_alloc;
1174                 struct Vmxnet3_RxDesc *rxd;
1175                 u32 idx, ring_idx;
1176                 struct vmxnet3_cmd_ring *ring = NULL;
1177                 if (num_rxd >= quota) {
1178                         /* we may stop even before we see the EOP desc of
1179                          * the current pkt
1180                          */
1181                         break;
1182                 }
1183                 num_rxd++;
1184                 BUG_ON(rcd->rqID != rq->qid && rcd->rqID != rq->qid2);
1185                 idx = rcd->rxdIdx;
1186                 ring_idx = rcd->rqID < adapter->num_rx_queues ? 0 : 1;
1187                 ring = rq->rx_ring + ring_idx;
1188                 vmxnet3_getRxDesc(rxd, &rq->rx_ring[ring_idx].base[idx].rxd,
1189                                   &rxCmdDesc);
1190                 rbi = rq->buf_info[ring_idx] + idx;
1191 
1192                 BUG_ON(rxd->addr != rbi->dma_addr ||
1193                        rxd->len != rbi->len);
1194 
1195                 if (unlikely(rcd->eop && rcd->err)) {
1196                         vmxnet3_rx_error(rq, rcd, ctx, adapter);
1197                         goto rcd_done;
1198                 }
1199 
1200                 if (rcd->sop) { /* first buf of the pkt */
1201                         BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_HEAD ||
1202                                rcd->rqID != rq->qid);
1203 
1204                         BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_SKB);
1205                         BUG_ON(ctx->skb != NULL || rbi->skb == NULL);
1206 
1207                         if (unlikely(rcd->len == 0)) {
1208                                 /* Pretend the rx buffer is skipped. */
1209                                 BUG_ON(!(rcd->sop && rcd->eop));
1210                                 netdev_dbg(adapter->netdev,
1211                                         "rxRing[%u][%u] 0 length\n",
1212                                         ring_idx, idx);
1213                                 goto rcd_done;
1214                         }
1215 
1216                         skip_page_frags = false;
1217                         ctx->skb = rbi->skb;
1218                         new_skb = netdev_alloc_skb_ip_align(adapter->netdev,
1219                                                             rbi->len);
1220                         if (new_skb == NULL) {
1221                                 /* Skb allocation failed, do not handover this
1222                                  * skb to stack. Reuse it. Drop the existing pkt
1223                                  */
1224                                 rq->stats.rx_buf_alloc_failure++;
1225                                 ctx->skb = NULL;
1226                                 rq->stats.drop_total++;
1227                                 skip_page_frags = true;
1228                                 goto rcd_done;
1229                         }
1230 
1231                         dma_unmap_single(&adapter->pdev->dev, rbi->dma_addr,
1232                                          rbi->len,
1233                                          PCI_DMA_FROMDEVICE);
1234 
1235 #ifdef VMXNET3_RSS
1236                         if (rcd->rssType != VMXNET3_RCD_RSS_TYPE_NONE &&
1237                             (adapter->netdev->features & NETIF_F_RXHASH))
1238                                 skb_set_hash(ctx->skb,
1239                                              le32_to_cpu(rcd->rssHash),
1240                                              PKT_HASH_TYPE_L3);
1241 #endif
1242                         skb_put(ctx->skb, rcd->len);
1243 
1244                         /* Immediate refill */
1245                         rbi->skb = new_skb;
1246                         rbi->dma_addr = dma_map_single(&adapter->pdev->dev,
1247                                                        rbi->skb->data, rbi->len,
1248                                                        PCI_DMA_FROMDEVICE);
1249                         rxd->addr = cpu_to_le64(rbi->dma_addr);
1250                         rxd->len = rbi->len;
1251 
1252                 } else {
1253                         BUG_ON(ctx->skb == NULL && !skip_page_frags);
1254 
1255                         /* non SOP buffer must be type 1 in most cases */
1256                         BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE);
1257                         BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_BODY);
1258 
1259                         /* If an sop buffer was dropped, skip all
1260                          * following non-sop fragments. They will be reused.
1261                          */
1262                         if (skip_page_frags)
1263                                 goto rcd_done;
1264 
1265                         new_page = alloc_page(GFP_ATOMIC);
1266                         if (unlikely(new_page == NULL)) {
1267                                 /* Replacement page frag could not be allocated.
1268                                  * Reuse this page. Drop the pkt and free the
1269                                  * skb which contained this page as a frag. Skip
1270                                  * processing all the following non-sop frags.
1271                                  */
1272                                 rq->stats.rx_buf_alloc_failure++;
1273                                 dev_kfree_skb(ctx->skb);
1274                                 ctx->skb = NULL;
1275                                 skip_page_frags = true;
1276                                 goto rcd_done;
1277                         }
1278 
1279                         if (rcd->len) {
1280                                 dma_unmap_page(&adapter->pdev->dev,
1281                                                rbi->dma_addr, rbi->len,
1282                                                PCI_DMA_FROMDEVICE);
1283 
1284                                 vmxnet3_append_frag(ctx->skb, rcd, rbi);
1285                         }
1286 
1287                         /* Immediate refill */
1288                         rbi->page = new_page;
1289                         rbi->dma_addr = dma_map_page(&adapter->pdev->dev,
1290                                                      rbi->page,
1291                                                      0, PAGE_SIZE,
1292                                                      PCI_DMA_FROMDEVICE);
1293                         rxd->addr = cpu_to_le64(rbi->dma_addr);
1294                         rxd->len = rbi->len;
1295                 }
1296 
1297 
1298                 skb = ctx->skb;
1299                 if (rcd->eop) {
1300                         skb->len += skb->data_len;
1301 
1302                         vmxnet3_rx_csum(adapter, skb,
1303                                         (union Vmxnet3_GenericDesc *)rcd);
1304                         skb->protocol = eth_type_trans(skb, adapter->netdev);
1305 
1306                         if (unlikely(rcd->ts))
1307                                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rcd->tci);
1308 
1309                         if (adapter->netdev->features & NETIF_F_LRO)
1310                                 netif_receive_skb(skb);
1311                         else
1312                                 napi_gro_receive(&rq->napi, skb);
1313 
1314                         ctx->skb = NULL;
1315                 }
1316 
1317 rcd_done:
1318                 /* device may have skipped some rx descs */
1319                 ring->next2comp = idx;
1320                 num_to_alloc = vmxnet3_cmd_ring_desc_avail(ring);
1321                 ring = rq->rx_ring + ring_idx;
1322                 while (num_to_alloc) {
1323                         vmxnet3_getRxDesc(rxd, &ring->base[ring->next2fill].rxd,
1324                                           &rxCmdDesc);
1325                         BUG_ON(!rxd->addr);
1326 
1327                         /* Recv desc is ready to be used by the device */
1328                         rxd->gen = ring->gen;
1329                         vmxnet3_cmd_ring_adv_next2fill(ring);
1330                         num_to_alloc--;
1331                 }
1332 
1333                 /* if needed, update the register */
1334                 if (unlikely(rq->shared->updateRxProd)) {
1335                         VMXNET3_WRITE_BAR0_REG(adapter,
1336                                                rxprod_reg[ring_idx] + rq->qid * 8,
1337                                                ring->next2fill);
1338                 }
1339 
1340                 vmxnet3_comp_ring_adv_next2proc(&rq->comp_ring);
1341                 vmxnet3_getRxComp(rcd,
1342                                   &rq->comp_ring.base[rq->comp_ring.next2proc].rcd, &rxComp);
1343         }
1344 
1345         return num_rxd;
1346 }
1347 
1348 
1349 static void
1350 vmxnet3_rq_cleanup(struct vmxnet3_rx_queue *rq,
1351                    struct vmxnet3_adapter *adapter)
1352 {
1353         u32 i, ring_idx;
1354         struct Vmxnet3_RxDesc *rxd;
1355 
1356         for (ring_idx = 0; ring_idx < 2; ring_idx++) {
1357                 for (i = 0; i < rq->rx_ring[ring_idx].size; i++) {
1358 #ifdef __BIG_ENDIAN_BITFIELD
1359                         struct Vmxnet3_RxDesc rxDesc;
1360 #endif
1361                         vmxnet3_getRxDesc(rxd,
1362                                 &rq->rx_ring[ring_idx].base[i].rxd, &rxDesc);
1363 
1364                         if (rxd->btype == VMXNET3_RXD_BTYPE_HEAD &&
1365                                         rq->buf_info[ring_idx][i].skb) {
1366                                 dma_unmap_single(&adapter->pdev->dev, rxd->addr,
1367                                                  rxd->len, PCI_DMA_FROMDEVICE);
1368                                 dev_kfree_skb(rq->buf_info[ring_idx][i].skb);
1369                                 rq->buf_info[ring_idx][i].skb = NULL;
1370                         } else if (rxd->btype == VMXNET3_RXD_BTYPE_BODY &&
1371                                         rq->buf_info[ring_idx][i].page) {
1372                                 dma_unmap_page(&adapter->pdev->dev, rxd->addr,
1373                                                rxd->len, PCI_DMA_FROMDEVICE);
1374                                 put_page(rq->buf_info[ring_idx][i].page);
1375                                 rq->buf_info[ring_idx][i].page = NULL;
1376                         }
1377                 }
1378 
1379                 rq->rx_ring[ring_idx].gen = VMXNET3_INIT_GEN;
1380                 rq->rx_ring[ring_idx].next2fill =
1381                                         rq->rx_ring[ring_idx].next2comp = 0;
1382         }
1383 
1384         rq->comp_ring.gen = VMXNET3_INIT_GEN;
1385         rq->comp_ring.next2proc = 0;
1386 }
1387 
1388 
1389 static void
1390 vmxnet3_rq_cleanup_all(struct vmxnet3_adapter *adapter)
1391 {
1392         int i;
1393 
1394         for (i = 0; i < adapter->num_rx_queues; i++)
1395                 vmxnet3_rq_cleanup(&adapter->rx_queue[i], adapter);
1396 }
1397 
1398 
1399 static void vmxnet3_rq_destroy(struct vmxnet3_rx_queue *rq,
1400                                struct vmxnet3_adapter *adapter)
1401 {
1402         int i;
1403         int j;
1404 
1405         /* all rx buffers must have already been freed */
1406         for (i = 0; i < 2; i++) {
1407                 if (rq->buf_info[i]) {
1408                         for (j = 0; j < rq->rx_ring[i].size; j++)
1409                                 BUG_ON(rq->buf_info[i][j].page != NULL);
1410                 }
1411         }
1412 
1413 
1414         for (i = 0; i < 2; i++) {
1415                 if (rq->rx_ring[i].base) {
1416                         dma_free_coherent(&adapter->pdev->dev,
1417                                           rq->rx_ring[i].size
1418                                           * sizeof(struct Vmxnet3_RxDesc),
1419                                           rq->rx_ring[i].base,
1420                                           rq->rx_ring[i].basePA);
1421                         rq->rx_ring[i].base = NULL;
1422                 }
1423                 rq->buf_info[i] = NULL;
1424         }
1425 
1426         if (rq->comp_ring.base) {
1427                 dma_free_coherent(&adapter->pdev->dev, rq->comp_ring.size
1428                                   * sizeof(struct Vmxnet3_RxCompDesc),
1429                                   rq->comp_ring.base, rq->comp_ring.basePA);
1430                 rq->comp_ring.base = NULL;
1431         }
1432 
1433         if (rq->buf_info[0]) {
1434                 size_t sz = sizeof(struct vmxnet3_rx_buf_info) *
1435                         (rq->rx_ring[0].size + rq->rx_ring[1].size);
1436                 dma_free_coherent(&adapter->pdev->dev, sz, rq->buf_info[0],
1437                                   rq->buf_info_pa);
1438         }
1439 }
1440 
1441 
1442 static int
1443 vmxnet3_rq_init(struct vmxnet3_rx_queue *rq,
1444                 struct vmxnet3_adapter  *adapter)
1445 {
1446         int i;
1447 
1448         /* initialize buf_info */
1449         for (i = 0; i < rq->rx_ring[0].size; i++) {
1450 
1451                 /* 1st buf for a pkt is skbuff */
1452                 if (i % adapter->rx_buf_per_pkt == 0) {
1453                         rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_SKB;
1454                         rq->buf_info[0][i].len = adapter->skb_buf_size;
1455                 } else { /* subsequent bufs for a pkt is frag */
1456                         rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_PAGE;
1457                         rq->buf_info[0][i].len = PAGE_SIZE;
1458                 }
1459         }
1460         for (i = 0; i < rq->rx_ring[1].size; i++) {
1461                 rq->buf_info[1][i].buf_type = VMXNET3_RX_BUF_PAGE;
1462                 rq->buf_info[1][i].len = PAGE_SIZE;
1463         }
1464 
1465         /* reset internal state and allocate buffers for both rings */
1466         for (i = 0; i < 2; i++) {
1467                 rq->rx_ring[i].next2fill = rq->rx_ring[i].next2comp = 0;
1468 
1469                 memset(rq->rx_ring[i].base, 0, rq->rx_ring[i].size *
1470                        sizeof(struct Vmxnet3_RxDesc));
1471                 rq->rx_ring[i].gen = VMXNET3_INIT_GEN;
1472         }
1473         if (vmxnet3_rq_alloc_rx_buf(rq, 0, rq->rx_ring[0].size - 1,
1474                                     adapter) == 0) {
1475                 /* at least has 1 rx buffer for the 1st ring */
1476                 return -ENOMEM;
1477         }
1478         vmxnet3_rq_alloc_rx_buf(rq, 1, rq->rx_ring[1].size - 1, adapter);
1479 
1480         /* reset the comp ring */
1481         rq->comp_ring.next2proc = 0;
1482         memset(rq->comp_ring.base, 0, rq->comp_ring.size *
1483                sizeof(struct Vmxnet3_RxCompDesc));
1484         rq->comp_ring.gen = VMXNET3_INIT_GEN;
1485 
1486         /* reset rxctx */
1487         rq->rx_ctx.skb = NULL;
1488 
1489         /* stats are not reset */
1490         return 0;
1491 }
1492 
1493 
1494 static int
1495 vmxnet3_rq_init_all(struct vmxnet3_adapter *adapter)
1496 {
1497         int i, err = 0;
1498 
1499         for (i = 0; i < adapter->num_rx_queues; i++) {
1500                 err = vmxnet3_rq_init(&adapter->rx_queue[i], adapter);
1501                 if (unlikely(err)) {
1502                         dev_err(&adapter->netdev->dev, "%s: failed to "
1503                                 "initialize rx queue%i\n",
1504                                 adapter->netdev->name, i);
1505                         break;
1506                 }
1507         }
1508         return err;
1509 
1510 }
1511 
1512 
1513 static int
1514 vmxnet3_rq_create(struct vmxnet3_rx_queue *rq, struct vmxnet3_adapter *adapter)
1515 {
1516         int i;
1517         size_t sz;
1518         struct vmxnet3_rx_buf_info *bi;
1519 
1520         for (i = 0; i < 2; i++) {
1521 
1522                 sz = rq->rx_ring[i].size * sizeof(struct Vmxnet3_RxDesc);
1523                 rq->rx_ring[i].base = dma_alloc_coherent(
1524                                                 &adapter->pdev->dev, sz,
1525                                                 &rq->rx_ring[i].basePA,
1526                                                 GFP_KERNEL);
1527                 if (!rq->rx_ring[i].base) {
1528                         netdev_err(adapter->netdev,
1529                                    "failed to allocate rx ring %d\n", i);
1530                         goto err;
1531                 }
1532         }
1533 
1534         sz = rq->comp_ring.size * sizeof(struct Vmxnet3_RxCompDesc);
1535         rq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev, sz,
1536                                                 &rq->comp_ring.basePA,
1537                                                 GFP_KERNEL);
1538         if (!rq->comp_ring.base) {
1539                 netdev_err(adapter->netdev, "failed to allocate rx comp ring\n");
1540                 goto err;
1541         }
1542 
1543         sz = sizeof(struct vmxnet3_rx_buf_info) * (rq->rx_ring[0].size +
1544                                                    rq->rx_ring[1].size);
1545         bi = dma_zalloc_coherent(&adapter->pdev->dev, sz, &rq->buf_info_pa,
1546                                  GFP_KERNEL);
1547         if (!bi)
1548                 goto err;
1549 
1550         rq->buf_info[0] = bi;
1551         rq->buf_info[1] = bi + rq->rx_ring[0].size;
1552 
1553         return 0;
1554 
1555 err:
1556         vmxnet3_rq_destroy(rq, adapter);
1557         return -ENOMEM;
1558 }
1559 
1560 
1561 static int
1562 vmxnet3_rq_create_all(struct vmxnet3_adapter *adapter)
1563 {
1564         int i, err = 0;
1565 
1566         for (i = 0; i < adapter->num_rx_queues; i++) {
1567                 err = vmxnet3_rq_create(&adapter->rx_queue[i], adapter);
1568                 if (unlikely(err)) {
1569                         dev_err(&adapter->netdev->dev,
1570                                 "%s: failed to create rx queue%i\n",
1571                                 adapter->netdev->name, i);
1572                         goto err_out;
1573                 }
1574         }
1575         return err;
1576 err_out:
1577         vmxnet3_rq_destroy_all(adapter);
1578         return err;
1579 
1580 }
1581 
1582 /* Multiple queue aware polling function for tx and rx */
1583 
1584 static int
1585 vmxnet3_do_poll(struct vmxnet3_adapter *adapter, int budget)
1586 {
1587         int rcd_done = 0, i;
1588         if (unlikely(adapter->shared->ecr))
1589                 vmxnet3_process_events(adapter);
1590         for (i = 0; i < adapter->num_tx_queues; i++)
1591                 vmxnet3_tq_tx_complete(&adapter->tx_queue[i], adapter);
1592 
1593         for (i = 0; i < adapter->num_rx_queues; i++)
1594                 rcd_done += vmxnet3_rq_rx_complete(&adapter->rx_queue[i],
1595                                                    adapter, budget);
1596         return rcd_done;
1597 }
1598 
1599 
1600 static int
1601 vmxnet3_poll(struct napi_struct *napi, int budget)
1602 {
1603         struct vmxnet3_rx_queue *rx_queue = container_of(napi,
1604                                           struct vmxnet3_rx_queue, napi);
1605         int rxd_done;
1606 
1607         rxd_done = vmxnet3_do_poll(rx_queue->adapter, budget);
1608 
1609         if (rxd_done < budget) {
1610                 napi_complete(napi);
1611                 vmxnet3_enable_all_intrs(rx_queue->adapter);
1612         }
1613         return rxd_done;
1614 }
1615 
1616 /*
1617  * NAPI polling function for MSI-X mode with multiple Rx queues
1618  * Returns the # of the NAPI credit consumed (# of rx descriptors processed)
1619  */
1620 
1621 static int
1622 vmxnet3_poll_rx_only(struct napi_struct *napi, int budget)
1623 {
1624         struct vmxnet3_rx_queue *rq = container_of(napi,
1625                                                 struct vmxnet3_rx_queue, napi);
1626         struct vmxnet3_adapter *adapter = rq->adapter;
1627         int rxd_done;
1628 
1629         /* When sharing interrupt with corresponding tx queue, process
1630          * tx completions in that queue as well
1631          */
1632         if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE) {
1633                 struct vmxnet3_tx_queue *tq =
1634                                 &adapter->tx_queue[rq - adapter->rx_queue];
1635                 vmxnet3_tq_tx_complete(tq, adapter);
1636         }
1637 
1638         rxd_done = vmxnet3_rq_rx_complete(rq, adapter, budget);
1639 
1640         if (rxd_done < budget) {
1641                 napi_complete(napi);
1642                 vmxnet3_enable_intr(adapter, rq->comp_ring.intr_idx);
1643         }
1644         return rxd_done;
1645 }
1646 
1647 
1648 #ifdef CONFIG_PCI_MSI
1649 
1650 /*
1651  * Handle completion interrupts on tx queues
1652  * Returns whether or not the intr is handled
1653  */
1654 
1655 static irqreturn_t
1656 vmxnet3_msix_tx(int irq, void *data)
1657 {
1658         struct vmxnet3_tx_queue *tq = data;
1659         struct vmxnet3_adapter *adapter = tq->adapter;
1660 
1661         if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1662                 vmxnet3_disable_intr(adapter, tq->comp_ring.intr_idx);
1663 
1664         /* Handle the case where only one irq is allocate for all tx queues */
1665         if (adapter->share_intr == VMXNET3_INTR_TXSHARE) {
1666                 int i;
1667                 for (i = 0; i < adapter->num_tx_queues; i++) {
1668                         struct vmxnet3_tx_queue *txq = &adapter->tx_queue[i];
1669                         vmxnet3_tq_tx_complete(txq, adapter);
1670                 }
1671         } else {
1672                 vmxnet3_tq_tx_complete(tq, adapter);
1673         }
1674         vmxnet3_enable_intr(adapter, tq->comp_ring.intr_idx);
1675 
1676         return IRQ_HANDLED;
1677 }
1678 
1679 
1680 /*
1681  * Handle completion interrupts on rx queues. Returns whether or not the
1682  * intr is handled
1683  */
1684 
1685 static irqreturn_t
1686 vmxnet3_msix_rx(int irq, void *data)
1687 {
1688         struct vmxnet3_rx_queue *rq = data;
1689         struct vmxnet3_adapter *adapter = rq->adapter;
1690 
1691         /* disable intr if needed */
1692         if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1693                 vmxnet3_disable_intr(adapter, rq->comp_ring.intr_idx);
1694         napi_schedule(&rq->napi);
1695 
1696         return IRQ_HANDLED;
1697 }
1698 
1699 /*
1700  *----------------------------------------------------------------------------
1701  *
1702  * vmxnet3_msix_event --
1703  *
1704  *    vmxnet3 msix event intr handler
1705  *
1706  * Result:
1707  *    whether or not the intr is handled
1708  *
1709  *----------------------------------------------------------------------------
1710  */
1711 
1712 static irqreturn_t
1713 vmxnet3_msix_event(int irq, void *data)
1714 {
1715         struct net_device *dev = data;
1716         struct vmxnet3_adapter *adapter = netdev_priv(dev);
1717 
1718         /* disable intr if needed */
1719         if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1720                 vmxnet3_disable_intr(adapter, adapter->intr.event_intr_idx);
1721 
1722         if (adapter->shared->ecr)
1723                 vmxnet3_process_events(adapter);
1724 
1725         vmxnet3_enable_intr(adapter, adapter->intr.event_intr_idx);
1726 
1727         return IRQ_HANDLED;
1728 }
1729 
1730 #endif /* CONFIG_PCI_MSI  */
1731 
1732 
1733 /* Interrupt handler for vmxnet3  */
1734 static irqreturn_t
1735 vmxnet3_intr(int irq, void *dev_id)
1736 {
1737         struct net_device *dev = dev_id;
1738         struct vmxnet3_adapter *adapter = netdev_priv(dev);
1739 
1740         if (adapter->intr.type == VMXNET3_IT_INTX) {
1741                 u32 icr = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_ICR);
1742                 if (unlikely(icr == 0))
1743                         /* not ours */
1744                         return IRQ_NONE;
1745         }
1746 
1747 
1748         /* disable intr if needed */
1749         if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1750                 vmxnet3_disable_all_intrs(adapter);
1751 
1752         napi_schedule(&adapter->rx_queue[0].napi);
1753 
1754         return IRQ_HANDLED;
1755 }
1756 
1757 #ifdef CONFIG_NET_POLL_CONTROLLER
1758 
1759 /* netpoll callback. */
1760 static void
1761 vmxnet3_netpoll(struct net_device *netdev)
1762 {
1763         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1764 
1765         switch (adapter->intr.type) {
1766 #ifdef CONFIG_PCI_MSI
1767         case VMXNET3_IT_MSIX: {
1768                 int i;
1769                 for (i = 0; i < adapter->num_rx_queues; i++)
1770                         vmxnet3_msix_rx(0, &adapter->rx_queue[i]);
1771                 break;
1772         }
1773 #endif
1774         case VMXNET3_IT_MSI:
1775         default:
1776                 vmxnet3_intr(0, adapter->netdev);
1777                 break;
1778         }
1779 
1780 }
1781 #endif  /* CONFIG_NET_POLL_CONTROLLER */
1782 
1783 static int
1784 vmxnet3_request_irqs(struct vmxnet3_adapter *adapter)
1785 {
1786         struct vmxnet3_intr *intr = &adapter->intr;
1787         int err = 0, i;
1788         int vector = 0;
1789 
1790 #ifdef CONFIG_PCI_MSI
1791         if (adapter->intr.type == VMXNET3_IT_MSIX) {
1792                 for (i = 0; i < adapter->num_tx_queues; i++) {
1793                         if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) {
1794                                 sprintf(adapter->tx_queue[i].name, "%s-tx-%d",
1795                                         adapter->netdev->name, vector);
1796                                 err = request_irq(
1797                                               intr->msix_entries[vector].vector,
1798                                               vmxnet3_msix_tx, 0,
1799                                               adapter->tx_queue[i].name,
1800                                               &adapter->tx_queue[i]);
1801                         } else {
1802                                 sprintf(adapter->tx_queue[i].name, "%s-rxtx-%d",
1803                                         adapter->netdev->name, vector);
1804                         }
1805                         if (err) {
1806                                 dev_err(&adapter->netdev->dev,
1807                                         "Failed to request irq for MSIX, %s, "
1808                                         "error %d\n",
1809                                         adapter->tx_queue[i].name, err);
1810                                 return err;
1811                         }
1812 
1813                         /* Handle the case where only 1 MSIx was allocated for
1814                          * all tx queues */
1815                         if (adapter->share_intr == VMXNET3_INTR_TXSHARE) {
1816                                 for (; i < adapter->num_tx_queues; i++)
1817                                         adapter->tx_queue[i].comp_ring.intr_idx
1818                                                                 = vector;
1819                                 vector++;
1820                                 break;
1821                         } else {
1822                                 adapter->tx_queue[i].comp_ring.intr_idx
1823                                                                 = vector++;
1824                         }
1825                 }
1826                 if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE)
1827                         vector = 0;
1828 
1829                 for (i = 0; i < adapter->num_rx_queues; i++) {
1830                         if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE)
1831                                 sprintf(adapter->rx_queue[i].name, "%s-rx-%d",
1832                                         adapter->netdev->name, vector);
1833                         else
1834                                 sprintf(adapter->rx_queue[i].name, "%s-rxtx-%d",
1835                                         adapter->netdev->name, vector);
1836                         err = request_irq(intr->msix_entries[vector].vector,
1837                                           vmxnet3_msix_rx, 0,
1838                                           adapter->rx_queue[i].name,
1839                                           &(adapter->rx_queue[i]));
1840                         if (err) {
1841                                 netdev_err(adapter->netdev,
1842                                            "Failed to request irq for MSIX, "
1843                                            "%s, error %d\n",
1844                                            adapter->rx_queue[i].name, err);
1845                                 return err;
1846                         }
1847 
1848                         adapter->rx_queue[i].comp_ring.intr_idx = vector++;
1849                 }
1850 
1851                 sprintf(intr->event_msi_vector_name, "%s-event-%d",
1852                         adapter->netdev->name, vector);
1853                 err = request_irq(intr->msix_entries[vector].vector,
1854                                   vmxnet3_msix_event, 0,
1855                                   intr->event_msi_vector_name, adapter->netdev);
1856                 intr->event_intr_idx = vector;
1857 
1858         } else if (intr->type == VMXNET3_IT_MSI) {
1859                 adapter->num_rx_queues = 1;
1860                 err = request_irq(adapter->pdev->irq, vmxnet3_intr, 0,
1861                                   adapter->netdev->name, adapter->netdev);
1862         } else {
1863 #endif
1864                 adapter->num_rx_queues = 1;
1865                 err = request_irq(adapter->pdev->irq, vmxnet3_intr,
1866                                   IRQF_SHARED, adapter->netdev->name,
1867                                   adapter->netdev);
1868 #ifdef CONFIG_PCI_MSI
1869         }
1870 #endif
1871         intr->num_intrs = vector + 1;
1872         if (err) {
1873                 netdev_err(adapter->netdev,
1874                            "Failed to request irq (intr type:%d), error %d\n",
1875                            intr->type, err);
1876         } else {
1877                 /* Number of rx queues will not change after this */
1878                 for (i = 0; i < adapter->num_rx_queues; i++) {
1879                         struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
1880                         rq->qid = i;
1881                         rq->qid2 = i + adapter->num_rx_queues;
1882                 }
1883 
1884 
1885 
1886                 /* init our intr settings */
1887                 for (i = 0; i < intr->num_intrs; i++)
1888                         intr->mod_levels[i] = UPT1_IML_ADAPTIVE;
1889                 if (adapter->intr.type != VMXNET3_IT_MSIX) {
1890                         adapter->intr.event_intr_idx = 0;
1891                         for (i = 0; i < adapter->num_tx_queues; i++)
1892                                 adapter->tx_queue[i].comp_ring.intr_idx = 0;
1893                         adapter->rx_queue[0].comp_ring.intr_idx = 0;
1894                 }
1895 
1896                 netdev_info(adapter->netdev,
1897                             "intr type %u, mode %u, %u vectors allocated\n",
1898                             intr->type, intr->mask_mode, intr->num_intrs);
1899         }
1900 
1901         return err;
1902 }
1903 
1904 
1905 static void
1906 vmxnet3_free_irqs(struct vmxnet3_adapter *adapter)
1907 {
1908         struct vmxnet3_intr *intr = &adapter->intr;
1909         BUG_ON(intr->type == VMXNET3_IT_AUTO || intr->num_intrs <= 0);
1910 
1911         switch (intr->type) {
1912 #ifdef CONFIG_PCI_MSI
1913         case VMXNET3_IT_MSIX:
1914         {
1915                 int i, vector = 0;
1916 
1917                 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) {
1918                         for (i = 0; i < adapter->num_tx_queues; i++) {
1919                                 free_irq(intr->msix_entries[vector++].vector,
1920                                          &(adapter->tx_queue[i]));
1921                                 if (adapter->share_intr == VMXNET3_INTR_TXSHARE)
1922                                         break;
1923                         }
1924                 }
1925 
1926                 for (i = 0; i < adapter->num_rx_queues; i++) {
1927                         free_irq(intr->msix_entries[vector++].vector,
1928                                  &(adapter->rx_queue[i]));
1929                 }
1930 
1931                 free_irq(intr->msix_entries[vector].vector,
1932                          adapter->netdev);
1933                 BUG_ON(vector >= intr->num_intrs);
1934                 break;
1935         }
1936 #endif
1937         case VMXNET3_IT_MSI:
1938                 free_irq(adapter->pdev->irq, adapter->netdev);
1939                 break;
1940         case VMXNET3_IT_INTX:
1941                 free_irq(adapter->pdev->irq, adapter->netdev);
1942                 break;
1943         default:
1944                 BUG();
1945         }
1946 }
1947 
1948 
1949 static void
1950 vmxnet3_restore_vlan(struct vmxnet3_adapter *adapter)
1951 {
1952         u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
1953         u16 vid;
1954 
1955         /* allow untagged pkts */
1956         VMXNET3_SET_VFTABLE_ENTRY(vfTable, 0);
1957 
1958         for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
1959                 VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid);
1960 }
1961 
1962 
1963 static int
1964 vmxnet3_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
1965 {
1966         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1967 
1968         if (!(netdev->flags & IFF_PROMISC)) {
1969                 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
1970                 unsigned long flags;
1971 
1972                 VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid);
1973                 spin_lock_irqsave(&adapter->cmd_lock, flags);
1974                 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
1975                                        VMXNET3_CMD_UPDATE_VLAN_FILTERS);
1976                 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
1977         }
1978 
1979         set_bit(vid, adapter->active_vlans);
1980 
1981         return 0;
1982 }
1983 
1984 
1985 static int
1986 vmxnet3_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid)
1987 {
1988         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1989 
1990         if (!(netdev->flags & IFF_PROMISC)) {
1991                 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
1992                 unsigned long flags;
1993 
1994                 VMXNET3_CLEAR_VFTABLE_ENTRY(vfTable, vid);
1995                 spin_lock_irqsave(&adapter->cmd_lock, flags);
1996                 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
1997                                        VMXNET3_CMD_UPDATE_VLAN_FILTERS);
1998                 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
1999         }
2000 
2001         clear_bit(vid, adapter->active_vlans);
2002 
2003         return 0;
2004 }
2005 
2006 
2007 static u8 *
2008 vmxnet3_copy_mc(struct net_device *netdev)
2009 {
2010         u8 *buf = NULL;
2011         u32 sz = netdev_mc_count(netdev) * ETH_ALEN;
2012 
2013         /* struct Vmxnet3_RxFilterConf.mfTableLen is u16. */
2014         if (sz <= 0xffff) {
2015                 /* We may be called with BH disabled */
2016                 buf = kmalloc(sz, GFP_ATOMIC);
2017                 if (buf) {
2018                         struct netdev_hw_addr *ha;
2019                         int i = 0;
2020 
2021                         netdev_for_each_mc_addr(ha, netdev)
2022                                 memcpy(buf + i++ * ETH_ALEN, ha->addr,
2023                                        ETH_ALEN);
2024                 }
2025         }
2026         return buf;
2027 }
2028 
2029 
2030 static void
2031 vmxnet3_set_mc(struct net_device *netdev)
2032 {
2033         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2034         unsigned long flags;
2035         struct Vmxnet3_RxFilterConf *rxConf =
2036                                         &adapter->shared->devRead.rxFilterConf;
2037         u8 *new_table = NULL;
2038         dma_addr_t new_table_pa = 0;
2039         u32 new_mode = VMXNET3_RXM_UCAST;
2040 
2041         if (netdev->flags & IFF_PROMISC) {
2042                 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
2043                 memset(vfTable, 0, VMXNET3_VFT_SIZE * sizeof(*vfTable));
2044 
2045                 new_mode |= VMXNET3_RXM_PROMISC;
2046         } else {
2047                 vmxnet3_restore_vlan(adapter);
2048         }
2049 
2050         if (netdev->flags & IFF_BROADCAST)
2051                 new_mode |= VMXNET3_RXM_BCAST;
2052 
2053         if (netdev->flags & IFF_ALLMULTI)
2054                 new_mode |= VMXNET3_RXM_ALL_MULTI;
2055         else
2056                 if (!netdev_mc_empty(netdev)) {
2057                         new_table = vmxnet3_copy_mc(netdev);
2058                         if (new_table) {
2059                                 new_mode |= VMXNET3_RXM_MCAST;
2060                                 rxConf->mfTableLen = cpu_to_le16(
2061                                         netdev_mc_count(netdev) * ETH_ALEN);
2062                                 new_table_pa = dma_map_single(
2063                                                         &adapter->pdev->dev,
2064                                                         new_table,
2065                                                         rxConf->mfTableLen,
2066                                                         PCI_DMA_TODEVICE);
2067                                 rxConf->mfTablePA = cpu_to_le64(new_table_pa);
2068                         } else {
2069                                 netdev_info(netdev, "failed to copy mcast list"
2070                                             ", setting ALL_MULTI\n");
2071                                 new_mode |= VMXNET3_RXM_ALL_MULTI;
2072                         }
2073                 }
2074 
2075 
2076         if (!(new_mode & VMXNET3_RXM_MCAST)) {
2077                 rxConf->mfTableLen = 0;
2078                 rxConf->mfTablePA = 0;
2079         }
2080 
2081         spin_lock_irqsave(&adapter->cmd_lock, flags);
2082         if (new_mode != rxConf->rxMode) {
2083                 rxConf->rxMode = cpu_to_le32(new_mode);
2084                 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2085                                        VMXNET3_CMD_UPDATE_RX_MODE);
2086                 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2087                                        VMXNET3_CMD_UPDATE_VLAN_FILTERS);
2088         }
2089 
2090         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2091                                VMXNET3_CMD_UPDATE_MAC_FILTERS);
2092         spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2093 
2094         if (new_table) {
2095                 dma_unmap_single(&adapter->pdev->dev, new_table_pa,
2096                                  rxConf->mfTableLen, PCI_DMA_TODEVICE);
2097                 kfree(new_table);
2098         }
2099 }
2100 
2101 void
2102 vmxnet3_rq_destroy_all(struct vmxnet3_adapter *adapter)
2103 {
2104         int i;
2105 
2106         for (i = 0; i < adapter->num_rx_queues; i++)
2107                 vmxnet3_rq_destroy(&adapter->rx_queue[i], adapter);
2108 }
2109 
2110 
2111 /*
2112  *   Set up driver_shared based on settings in adapter.
2113  */
2114 
2115 static void
2116 vmxnet3_setup_driver_shared(struct vmxnet3_adapter *adapter)
2117 {
2118         struct Vmxnet3_DriverShared *shared = adapter->shared;
2119         struct Vmxnet3_DSDevRead *devRead = &shared->devRead;
2120         struct Vmxnet3_TxQueueConf *tqc;
2121         struct Vmxnet3_RxQueueConf *rqc;
2122         int i;
2123 
2124         memset(shared, 0, sizeof(*shared));
2125 
2126         /* driver settings */
2127         shared->magic = cpu_to_le32(VMXNET3_REV1_MAGIC);
2128         devRead->misc.driverInfo.version = cpu_to_le32(
2129                                                 VMXNET3_DRIVER_VERSION_NUM);
2130         devRead->misc.driverInfo.gos.gosBits = (sizeof(void *) == 4 ?
2131                                 VMXNET3_GOS_BITS_32 : VMXNET3_GOS_BITS_64);
2132         devRead->misc.driverInfo.gos.gosType = VMXNET3_GOS_TYPE_LINUX;
2133         *((u32 *)&devRead->misc.driverInfo.gos) = cpu_to_le32(
2134                                 *((u32 *)&devRead->misc.driverInfo.gos));
2135         devRead->misc.driverInfo.vmxnet3RevSpt = cpu_to_le32(1);
2136         devRead->misc.driverInfo.uptVerSpt = cpu_to_le32(1);
2137 
2138         devRead->misc.ddPA = cpu_to_le64(adapter->adapter_pa);
2139         devRead->misc.ddLen = cpu_to_le32(sizeof(struct vmxnet3_adapter));
2140 
2141         /* set up feature flags */
2142         if (adapter->netdev->features & NETIF_F_RXCSUM)
2143                 devRead->misc.uptFeatures |= UPT1_F_RXCSUM;
2144 
2145         if (adapter->netdev->features & NETIF_F_LRO) {
2146                 devRead->misc.uptFeatures |= UPT1_F_LRO;
2147                 devRead->misc.maxNumRxSG = cpu_to_le16(1 + MAX_SKB_FRAGS);
2148         }
2149         if (adapter->netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
2150                 devRead->misc.uptFeatures |= UPT1_F_RXVLAN;
2151 
2152         devRead->misc.mtu = cpu_to_le32(adapter->netdev->mtu);
2153         devRead->misc.queueDescPA = cpu_to_le64(adapter->queue_desc_pa);
2154         devRead->misc.queueDescLen = cpu_to_le32(
2155                 adapter->num_tx_queues * sizeof(struct Vmxnet3_TxQueueDesc) +
2156                 adapter->num_rx_queues * sizeof(struct Vmxnet3_RxQueueDesc));
2157 
2158         /* tx queue settings */
2159         devRead->misc.numTxQueues =  adapter->num_tx_queues;
2160         for (i = 0; i < adapter->num_tx_queues; i++) {
2161                 struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i];
2162                 BUG_ON(adapter->tx_queue[i].tx_ring.base == NULL);
2163                 tqc = &adapter->tqd_start[i].conf;
2164                 tqc->txRingBasePA   = cpu_to_le64(tq->tx_ring.basePA);
2165                 tqc->dataRingBasePA = cpu_to_le64(tq->data_ring.basePA);
2166                 tqc->compRingBasePA = cpu_to_le64(tq->comp_ring.basePA);
2167                 tqc->ddPA           = cpu_to_le64(tq->buf_info_pa);
2168                 tqc->txRingSize     = cpu_to_le32(tq->tx_ring.size);
2169                 tqc->dataRingSize   = cpu_to_le32(tq->data_ring.size);
2170                 tqc->compRingSize   = cpu_to_le32(tq->comp_ring.size);
2171                 tqc->ddLen          = cpu_to_le32(
2172                                         sizeof(struct vmxnet3_tx_buf_info) *
2173                                         tqc->txRingSize);
2174                 tqc->intrIdx        = tq->comp_ring.intr_idx;
2175         }
2176 
2177         /* rx queue settings */
2178         devRead->misc.numRxQueues = adapter->num_rx_queues;
2179         for (i = 0; i < adapter->num_rx_queues; i++) {
2180                 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
2181                 rqc = &adapter->rqd_start[i].conf;
2182                 rqc->rxRingBasePA[0] = cpu_to_le64(rq->rx_ring[0].basePA);
2183                 rqc->rxRingBasePA[1] = cpu_to_le64(rq->rx_ring[1].basePA);
2184                 rqc->compRingBasePA  = cpu_to_le64(rq->comp_ring.basePA);
2185                 rqc->ddPA            = cpu_to_le64(rq->buf_info_pa);
2186                 rqc->rxRingSize[0]   = cpu_to_le32(rq->rx_ring[0].size);
2187                 rqc->rxRingSize[1]   = cpu_to_le32(rq->rx_ring[1].size);
2188                 rqc->compRingSize    = cpu_to_le32(rq->comp_ring.size);
2189                 rqc->ddLen           = cpu_to_le32(
2190                                         sizeof(struct vmxnet3_rx_buf_info) *
2191                                         (rqc->rxRingSize[0] +
2192                                          rqc->rxRingSize[1]));
2193                 rqc->intrIdx         = rq->comp_ring.intr_idx;
2194         }
2195 
2196 #ifdef VMXNET3_RSS
2197         memset(adapter->rss_conf, 0, sizeof(*adapter->rss_conf));
2198 
2199         if (adapter->rss) {
2200                 struct UPT1_RSSConf *rssConf = adapter->rss_conf;
2201                 static const uint8_t rss_key[UPT1_RSS_MAX_KEY_SIZE] = {
2202                         0x3b, 0x56, 0xd1, 0x56, 0x13, 0x4a, 0xe7, 0xac,
2203                         0xe8, 0x79, 0x09, 0x75, 0xe8, 0x65, 0x79, 0x28,
2204                         0x35, 0x12, 0xb9, 0x56, 0x7c, 0x76, 0x4b, 0x70,
2205                         0xd8, 0x56, 0xa3, 0x18, 0x9b, 0x0a, 0xee, 0xf3,
2206                         0x96, 0xa6, 0x9f, 0x8f, 0x9e, 0x8c, 0x90, 0xc9,
2207                 };
2208 
2209                 devRead->misc.uptFeatures |= UPT1_F_RSS;
2210                 devRead->misc.numRxQueues = adapter->num_rx_queues;
2211                 rssConf->hashType = UPT1_RSS_HASH_TYPE_TCP_IPV4 |
2212                                     UPT1_RSS_HASH_TYPE_IPV4 |
2213                                     UPT1_RSS_HASH_TYPE_TCP_IPV6 |
2214                                     UPT1_RSS_HASH_TYPE_IPV6;
2215                 rssConf->hashFunc = UPT1_RSS_HASH_FUNC_TOEPLITZ;
2216                 rssConf->hashKeySize = UPT1_RSS_MAX_KEY_SIZE;
2217                 rssConf->indTableSize = VMXNET3_RSS_IND_TABLE_SIZE;
2218                 memcpy(rssConf->hashKey, rss_key, sizeof(rss_key));
2219 
2220                 for (i = 0; i < rssConf->indTableSize; i++)
2221                         rssConf->indTable[i] = ethtool_rxfh_indir_default(
2222                                 i, adapter->num_rx_queues);
2223 
2224                 devRead->rssConfDesc.confVer = 1;
2225                 devRead->rssConfDesc.confLen = cpu_to_le32(sizeof(*rssConf));
2226                 devRead->rssConfDesc.confPA =
2227                         cpu_to_le64(adapter->rss_conf_pa);
2228         }
2229 
2230 #endif /* VMXNET3_RSS */
2231 
2232         /* intr settings */
2233         devRead->intrConf.autoMask = adapter->intr.mask_mode ==
2234                                      VMXNET3_IMM_AUTO;
2235         devRead->intrConf.numIntrs = adapter->intr.num_intrs;
2236         for (i = 0; i < adapter->intr.num_intrs; i++)
2237                 devRead->intrConf.modLevels[i] = adapter->intr.mod_levels[i];
2238 
2239         devRead->intrConf.eventIntrIdx = adapter->intr.event_intr_idx;
2240         devRead->intrConf.intrCtrl |= cpu_to_le32(VMXNET3_IC_DISABLE_ALL);
2241 
2242         /* rx filter settings */
2243         devRead->rxFilterConf.rxMode = 0;
2244         vmxnet3_restore_vlan(adapter);
2245         vmxnet3_write_mac_addr(adapter, adapter->netdev->dev_addr);
2246 
2247         /* the rest are already zeroed */
2248 }
2249 
2250 
2251 int
2252 vmxnet3_activate_dev(struct vmxnet3_adapter *adapter)
2253 {
2254         int err, i;
2255         u32 ret;
2256         unsigned long flags;
2257 
2258         netdev_dbg(adapter->netdev, "%s: skb_buf_size %d, rx_buf_per_pkt %d,"
2259                 " ring sizes %u %u %u\n", adapter->netdev->name,
2260                 adapter->skb_buf_size, adapter->rx_buf_per_pkt,
2261                 adapter->tx_queue[0].tx_ring.size,
2262                 adapter->rx_queue[0].rx_ring[0].size,
2263                 adapter->rx_queue[0].rx_ring[1].size);
2264 
2265         vmxnet3_tq_init_all(adapter);
2266         err = vmxnet3_rq_init_all(adapter);
2267         if (err) {
2268                 netdev_err(adapter->netdev,
2269                            "Failed to init rx queue error %d\n", err);
2270                 goto rq_err;
2271         }
2272 
2273         err = vmxnet3_request_irqs(adapter);
2274         if (err) {
2275                 netdev_err(adapter->netdev,
2276                            "Failed to setup irq for error %d\n", err);
2277                 goto irq_err;
2278         }
2279 
2280         vmxnet3_setup_driver_shared(adapter);
2281 
2282         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, VMXNET3_GET_ADDR_LO(
2283                                adapter->shared_pa));
2284         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, VMXNET3_GET_ADDR_HI(
2285                                adapter->shared_pa));
2286         spin_lock_irqsave(&adapter->cmd_lock, flags);
2287         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2288                                VMXNET3_CMD_ACTIVATE_DEV);
2289         ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
2290         spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2291 
2292         if (ret != 0) {
2293                 netdev_err(adapter->netdev,
2294                            "Failed to activate dev: error %u\n", ret);
2295                 err = -EINVAL;
2296                 goto activate_err;
2297         }
2298 
2299         for (i = 0; i < adapter->num_rx_queues; i++) {
2300                 VMXNET3_WRITE_BAR0_REG(adapter,
2301                                 VMXNET3_REG_RXPROD + i * VMXNET3_REG_ALIGN,
2302                                 adapter->rx_queue[i].rx_ring[0].next2fill);
2303                 VMXNET3_WRITE_BAR0_REG(adapter, (VMXNET3_REG_RXPROD2 +
2304                                 (i * VMXNET3_REG_ALIGN)),
2305                                 adapter->rx_queue[i].rx_ring[1].next2fill);
2306         }
2307 
2308         /* Apply the rx filter settins last. */
2309         vmxnet3_set_mc(adapter->netdev);
2310 
2311         /*
2312          * Check link state when first activating device. It will start the
2313          * tx queue if the link is up.
2314          */
2315         vmxnet3_check_link(adapter, true);
2316         for (i = 0; i < adapter->num_rx_queues; i++)
2317                 napi_enable(&adapter->rx_queue[i].napi);
2318         vmxnet3_enable_all_intrs(adapter);
2319         clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);
2320         return 0;
2321 
2322 activate_err:
2323         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, 0);
2324         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, 0);
2325         vmxnet3_free_irqs(adapter);
2326 irq_err:
2327 rq_err:
2328         /* free up buffers we allocated */
2329         vmxnet3_rq_cleanup_all(adapter);
2330         return err;
2331 }
2332 
2333 
2334 void
2335 vmxnet3_reset_dev(struct vmxnet3_adapter *adapter)
2336 {
2337         unsigned long flags;
2338         spin_lock_irqsave(&adapter->cmd_lock, flags);
2339         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV);
2340         spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2341 }
2342 
2343 
2344 int
2345 vmxnet3_quiesce_dev(struct vmxnet3_adapter *adapter)
2346 {
2347         int i;
2348         unsigned long flags;
2349         if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state))
2350                 return 0;
2351 
2352 
2353         spin_lock_irqsave(&adapter->cmd_lock, flags);
2354         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2355                                VMXNET3_CMD_QUIESCE_DEV);
2356         spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2357         vmxnet3_disable_all_intrs(adapter);
2358 
2359         for (i = 0; i < adapter->num_rx_queues; i++)
2360                 napi_disable(&adapter->rx_queue[i].napi);
2361         netif_tx_disable(adapter->netdev);
2362         adapter->link_speed = 0;
2363         netif_carrier_off(adapter->netdev);
2364 
2365         vmxnet3_tq_cleanup_all(adapter);
2366         vmxnet3_rq_cleanup_all(adapter);
2367         vmxnet3_free_irqs(adapter);
2368         return 0;
2369 }
2370 
2371 
2372 static void
2373 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac)
2374 {
2375         u32 tmp;
2376 
2377         tmp = *(u32 *)mac;
2378         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACL, tmp);
2379 
2380         tmp = (mac[5] << 8) | mac[4];
2381         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACH, tmp);
2382 }
2383 
2384 
2385 static int
2386 vmxnet3_set_mac_addr(struct net_device *netdev, void *p)
2387 {
2388         struct sockaddr *addr = p;
2389         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2390 
2391         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
2392         vmxnet3_write_mac_addr(adapter, addr->sa_data);
2393 
2394         return 0;
2395 }
2396 
2397 
2398 /* ==================== initialization and cleanup routines ============ */
2399 
2400 static int
2401 vmxnet3_alloc_pci_resources(struct vmxnet3_adapter *adapter, bool *dma64)
2402 {
2403         int err;
2404         unsigned long mmio_start, mmio_len;
2405         struct pci_dev *pdev = adapter->pdev;
2406 
2407         err = pci_enable_device(pdev);
2408         if (err) {
2409                 dev_err(&pdev->dev, "Failed to enable adapter: error %d\n", err);
2410                 return err;
2411         }
2412 
2413         if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64)) == 0) {
2414                 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)) != 0) {
2415                         dev_err(&pdev->dev,
2416                                 "pci_set_consistent_dma_mask failed\n");
2417                         err = -EIO;
2418                         goto err_set_mask;
2419                 }
2420                 *dma64 = true;
2421         } else {
2422                 if (pci_set_dma_mask(pdev, DMA_BIT_MASK(32)) != 0) {
2423                         dev_err(&pdev->dev,
2424                                 "pci_set_dma_mask failed\n");
2425                         err = -EIO;
2426                         goto err_set_mask;
2427                 }
2428                 *dma64 = false;
2429         }
2430 
2431         err = pci_request_selected_regions(pdev, (1 << 2) - 1,
2432                                            vmxnet3_driver_name);
2433         if (err) {
2434                 dev_err(&pdev->dev,
2435                         "Failed to request region for adapter: error %d\n", err);
2436                 goto err_set_mask;
2437         }
2438 
2439         pci_set_master(pdev);
2440 
2441         mmio_start = pci_resource_start(pdev, 0);
2442         mmio_len = pci_resource_len(pdev, 0);
2443         adapter->hw_addr0 = ioremap(mmio_start, mmio_len);
2444         if (!adapter->hw_addr0) {
2445                 dev_err(&pdev->dev, "Failed to map bar0\n");
2446                 err = -EIO;
2447                 goto err_ioremap;
2448         }
2449 
2450         mmio_start = pci_resource_start(pdev, 1);
2451         mmio_len = pci_resource_len(pdev, 1);
2452         adapter->hw_addr1 = ioremap(mmio_start, mmio_len);
2453         if (!adapter->hw_addr1) {
2454                 dev_err(&pdev->dev, "Failed to map bar1\n");
2455                 err = -EIO;
2456                 goto err_bar1;
2457         }
2458         return 0;
2459 
2460 err_bar1:
2461         iounmap(adapter->hw_addr0);
2462 err_ioremap:
2463         pci_release_selected_regions(pdev, (1 << 2) - 1);
2464 err_set_mask:
2465         pci_disable_device(pdev);
2466         return err;
2467 }
2468 
2469 
2470 static void
2471 vmxnet3_free_pci_resources(struct vmxnet3_adapter *adapter)
2472 {
2473         BUG_ON(!adapter->pdev);
2474 
2475         iounmap(adapter->hw_addr0);
2476         iounmap(adapter->hw_addr1);
2477         pci_release_selected_regions(adapter->pdev, (1 << 2) - 1);
2478         pci_disable_device(adapter->pdev);
2479 }
2480 
2481 
2482 static void
2483 vmxnet3_adjust_rx_ring_size(struct vmxnet3_adapter *adapter)
2484 {
2485         size_t sz, i, ring0_size, ring1_size, comp_size;
2486         struct vmxnet3_rx_queue *rq = &adapter->rx_queue[0];
2487 
2488 
2489         if (adapter->netdev->mtu <= VMXNET3_MAX_SKB_BUF_SIZE -
2490                                     VMXNET3_MAX_ETH_HDR_SIZE) {
2491                 adapter->skb_buf_size = adapter->netdev->mtu +
2492                                         VMXNET3_MAX_ETH_HDR_SIZE;
2493                 if (adapter->skb_buf_size < VMXNET3_MIN_T0_BUF_SIZE)
2494                         adapter->skb_buf_size = VMXNET3_MIN_T0_BUF_SIZE;
2495 
2496                 adapter->rx_buf_per_pkt = 1;
2497         } else {
2498                 adapter->skb_buf_size = VMXNET3_MAX_SKB_BUF_SIZE;
2499                 sz = adapter->netdev->mtu - VMXNET3_MAX_SKB_BUF_SIZE +
2500                                             VMXNET3_MAX_ETH_HDR_SIZE;
2501                 adapter->rx_buf_per_pkt = 1 + (sz + PAGE_SIZE - 1) / PAGE_SIZE;
2502         }
2503 
2504         /*
2505          * for simplicity, force the ring0 size to be a multiple of
2506          * rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN
2507          */
2508         sz = adapter->rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN;
2509         ring0_size = adapter->rx_queue[0].rx_ring[0].size;
2510         ring0_size = (ring0_size + sz - 1) / sz * sz;
2511         ring0_size = min_t(u32, ring0_size, VMXNET3_RX_RING_MAX_SIZE /
2512                            sz * sz);
2513         ring1_size = adapter->rx_queue[0].rx_ring[1].size;
2514         comp_size = ring0_size + ring1_size;
2515 
2516         for (i = 0; i < adapter->num_rx_queues; i++) {
2517                 rq = &adapter->rx_queue[i];
2518                 rq->rx_ring[0].size = ring0_size;
2519                 rq->rx_ring[1].size = ring1_size;
2520                 rq->comp_ring.size = comp_size;
2521         }
2522 }
2523 
2524 
2525 int
2526 vmxnet3_create_queues(struct vmxnet3_adapter *adapter, u32 tx_ring_size,
2527                       u32 rx_ring_size, u32 rx_ring2_size)
2528 {
2529         int err = 0, i;
2530 
2531         for (i = 0; i < adapter->num_tx_queues; i++) {
2532                 struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i];
2533                 tq->tx_ring.size   = tx_ring_size;
2534                 tq->data_ring.size = tx_ring_size;
2535                 tq->comp_ring.size = tx_ring_size;
2536                 tq->shared = &adapter->tqd_start[i].ctrl;
2537                 tq->stopped = true;
2538                 tq->adapter = adapter;
2539                 tq->qid = i;
2540                 err = vmxnet3_tq_create(tq, adapter);
2541                 /*
2542                  * Too late to change num_tx_queues. We cannot do away with
2543                  * lesser number of queues than what we asked for
2544                  */
2545                 if (err)
2546                         goto queue_err;
2547         }
2548 
2549         adapter->rx_queue[0].rx_ring[0].size = rx_ring_size;
2550         adapter->rx_queue[0].rx_ring[1].size = rx_ring2_size;
2551         vmxnet3_adjust_rx_ring_size(adapter);
2552         for (i = 0; i < adapter->num_rx_queues; i++) {
2553                 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
2554                 /* qid and qid2 for rx queues will be assigned later when num
2555                  * of rx queues is finalized after allocating intrs */
2556                 rq->shared = &adapter->rqd_start[i].ctrl;
2557                 rq->adapter = adapter;
2558                 err = vmxnet3_rq_create(rq, adapter);
2559                 if (err) {
2560                         if (i == 0) {
2561                                 netdev_err(adapter->netdev,
2562                                            "Could not allocate any rx queues. "
2563                                            "Aborting.\n");
2564                                 goto queue_err;
2565                         } else {
2566                                 netdev_info(adapter->netdev,
2567                                             "Number of rx queues changed "
2568                                             "to : %d.\n", i);
2569                                 adapter->num_rx_queues = i;
2570                                 err = 0;
2571                                 break;
2572                         }
2573                 }
2574         }
2575         return err;
2576 queue_err:
2577         vmxnet3_tq_destroy_all(adapter);
2578         return err;
2579 }
2580 
2581 static int
2582 vmxnet3_open(struct net_device *netdev)
2583 {
2584         struct vmxnet3_adapter *adapter;
2585         int err, i;
2586 
2587         adapter = netdev_priv(netdev);
2588 
2589         for (i = 0; i < adapter->num_tx_queues; i++)
2590                 spin_lock_init(&adapter->tx_queue[i].tx_lock);
2591 
2592         err = vmxnet3_create_queues(adapter, VMXNET3_DEF_TX_RING_SIZE,
2593                                     VMXNET3_DEF_RX_RING_SIZE,
2594                                     VMXNET3_DEF_RX_RING_SIZE);
2595         if (err)
2596                 goto queue_err;
2597 
2598         err = vmxnet3_activate_dev(adapter);
2599         if (err)
2600                 goto activate_err;
2601 
2602         return 0;
2603 
2604 activate_err:
2605         vmxnet3_rq_destroy_all(adapter);
2606         vmxnet3_tq_destroy_all(adapter);
2607 queue_err:
2608         return err;
2609 }
2610 
2611 
2612 static int
2613 vmxnet3_close(struct net_device *netdev)
2614 {
2615         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2616 
2617         /*
2618          * Reset_work may be in the middle of resetting the device, wait for its
2619          * completion.
2620          */
2621         while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
2622                 msleep(1);
2623 
2624         vmxnet3_quiesce_dev(adapter);
2625 
2626         vmxnet3_rq_destroy_all(adapter);
2627         vmxnet3_tq_destroy_all(adapter);
2628 
2629         clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
2630 
2631 
2632         return 0;
2633 }
2634 
2635 
2636 void
2637 vmxnet3_force_close(struct vmxnet3_adapter *adapter)
2638 {
2639         int i;
2640 
2641         /*
2642          * we must clear VMXNET3_STATE_BIT_RESETTING, otherwise
2643          * vmxnet3_close() will deadlock.
2644          */
2645         BUG_ON(test_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state));
2646 
2647         /* we need to enable NAPI, otherwise dev_close will deadlock */
2648         for (i = 0; i < adapter->num_rx_queues; i++)
2649                 napi_enable(&adapter->rx_queue[i].napi);
2650         dev_close(adapter->netdev);
2651 }
2652 
2653 
2654 static int
2655 vmxnet3_change_mtu(struct net_device *netdev, int new_mtu)
2656 {
2657         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2658         int err = 0;
2659 
2660         if (new_mtu < VMXNET3_MIN_MTU || new_mtu > VMXNET3_MAX_MTU)
2661                 return -EINVAL;
2662 
2663         netdev->mtu = new_mtu;
2664 
2665         /*
2666          * Reset_work may be in the middle of resetting the device, wait for its
2667          * completion.
2668          */
2669         while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
2670                 msleep(1);
2671 
2672         if (netif_running(netdev)) {
2673                 vmxnet3_quiesce_dev(adapter);
2674                 vmxnet3_reset_dev(adapter);
2675 
2676                 /* we need to re-create the rx queue based on the new mtu */
2677                 vmxnet3_rq_destroy_all(adapter);
2678                 vmxnet3_adjust_rx_ring_size(adapter);
2679                 err = vmxnet3_rq_create_all(adapter);
2680                 if (err) {
2681                         netdev_err(netdev,
2682                                    "failed to re-create rx queues, "
2683                                    " error %d. Closing it.\n", err);
2684                         goto out;
2685                 }
2686 
2687                 err = vmxnet3_activate_dev(adapter);
2688                 if (err) {
2689                         netdev_err(netdev,
2690                                    "failed to re-activate, error %d. "
2691                                    "Closing it\n", err);
2692                         goto out;
2693                 }
2694         }
2695 
2696 out:
2697         clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
2698         if (err)
2699                 vmxnet3_force_close(adapter);
2700 
2701         return err;
2702 }
2703 
2704 
2705 static void
2706 vmxnet3_declare_features(struct vmxnet3_adapter *adapter, bool dma64)
2707 {
2708         struct net_device *netdev = adapter->netdev;
2709 
2710         netdev->hw_features = NETIF_F_SG | NETIF_F_RXCSUM |
2711                 NETIF_F_HW_CSUM | NETIF_F_HW_VLAN_CTAG_TX |
2712                 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_TSO | NETIF_F_TSO6 |
2713                 NETIF_F_LRO;
2714         if (dma64)
2715                 netdev->hw_features |= NETIF_F_HIGHDMA;
2716         netdev->vlan_features = netdev->hw_features &
2717                                 ~(NETIF_F_HW_VLAN_CTAG_TX |
2718                                   NETIF_F_HW_VLAN_CTAG_RX);
2719         netdev->features = netdev->hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;
2720 }
2721 
2722 
2723 static void
2724 vmxnet3_read_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac)
2725 {
2726         u32 tmp;
2727 
2728         tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACL);
2729         *(u32 *)mac = tmp;
2730 
2731         tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACH);
2732         mac[4] = tmp & 0xff;
2733         mac[5] = (tmp >> 8) & 0xff;
2734 }
2735 
2736 #ifdef CONFIG_PCI_MSI
2737 
2738 /*
2739  * Enable MSIx vectors.
2740  * Returns :
2741  *      VMXNET3_LINUX_MIN_MSIX_VECT when only minimum number of vectors required
2742  *       were enabled.
2743  *      number of vectors which were enabled otherwise (this number is greater
2744  *       than VMXNET3_LINUX_MIN_MSIX_VECT)
2745  */
2746 
2747 static int
2748 vmxnet3_acquire_msix_vectors(struct vmxnet3_adapter *adapter, int nvec)
2749 {
2750         int ret = pci_enable_msix_range(adapter->pdev,
2751                                         adapter->intr.msix_entries, nvec, nvec);
2752 
2753         if (ret == -ENOSPC && nvec > VMXNET3_LINUX_MIN_MSIX_VECT) {
2754                 dev_err(&adapter->netdev->dev,
2755                         "Failed to enable %d MSI-X, trying %d\n",
2756                         nvec, VMXNET3_LINUX_MIN_MSIX_VECT);
2757 
2758                 ret = pci_enable_msix_range(adapter->pdev,
2759                                             adapter->intr.msix_entries,
2760                                             VMXNET3_LINUX_MIN_MSIX_VECT,
2761                                             VMXNET3_LINUX_MIN_MSIX_VECT);
2762         }
2763 
2764         if (ret < 0) {
2765                 dev_err(&adapter->netdev->dev,
2766                         "Failed to enable MSI-X, error: %d\n", ret);
2767         }
2768 
2769         return ret;
2770 }
2771 
2772 
2773 #endif /* CONFIG_PCI_MSI */
2774 
2775 static void
2776 vmxnet3_alloc_intr_resources(struct vmxnet3_adapter *adapter)
2777 {
2778         u32 cfg;
2779         unsigned long flags;
2780 
2781         /* intr settings */
2782         spin_lock_irqsave(&adapter->cmd_lock, flags);
2783         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2784                                VMXNET3_CMD_GET_CONF_INTR);
2785         cfg = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
2786         spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2787         adapter->intr.type = cfg & 0x3;
2788         adapter->intr.mask_mode = (cfg >> 2) & 0x3;
2789 
2790         if (adapter->intr.type == VMXNET3_IT_AUTO) {
2791                 adapter->intr.type = VMXNET3_IT_MSIX;
2792         }
2793 
2794 #ifdef CONFIG_PCI_MSI
2795         if (adapter->intr.type == VMXNET3_IT_MSIX) {
2796                 int i, nvec;
2797 
2798                 nvec  = adapter->share_intr == VMXNET3_INTR_TXSHARE ?
2799                         1 : adapter->num_tx_queues;
2800                 nvec += adapter->share_intr == VMXNET3_INTR_BUDDYSHARE ?
2801                         0 : adapter->num_rx_queues;
2802                 nvec += 1;      /* for link event */
2803                 nvec = nvec > VMXNET3_LINUX_MIN_MSIX_VECT ?
2804                        nvec : VMXNET3_LINUX_MIN_MSIX_VECT;
2805 
2806                 for (i = 0; i < nvec; i++)
2807                         adapter->intr.msix_entries[i].entry = i;
2808 
2809                 nvec = vmxnet3_acquire_msix_vectors(adapter, nvec);
2810                 if (nvec < 0)
2811                         goto msix_err;
2812 
2813                 /* If we cannot allocate one MSIx vector per queue
2814                  * then limit the number of rx queues to 1
2815                  */
2816                 if (nvec == VMXNET3_LINUX_MIN_MSIX_VECT) {
2817                         if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE
2818                             || adapter->num_rx_queues != 1) {
2819                                 adapter->share_intr = VMXNET3_INTR_TXSHARE;
2820                                 netdev_err(adapter->netdev,
2821                                            "Number of rx queues : 1\n");
2822                                 adapter->num_rx_queues = 1;
2823                         }
2824                 }
2825 
2826                 adapter->intr.num_intrs = nvec;
2827                 return;
2828 
2829 msix_err:
2830                 /* If we cannot allocate MSIx vectors use only one rx queue */
2831                 dev_info(&adapter->pdev->dev,
2832                          "Failed to enable MSI-X, error %d. "
2833                          "Limiting #rx queues to 1, try MSI.\n", nvec);
2834 
2835                 adapter->intr.type = VMXNET3_IT_MSI;
2836         }
2837 
2838         if (adapter->intr.type == VMXNET3_IT_MSI) {
2839                 if (!pci_enable_msi(adapter->pdev)) {
2840                         adapter->num_rx_queues = 1;
2841                         adapter->intr.num_intrs = 1;
2842                         return;
2843                 }
2844         }
2845 #endif /* CONFIG_PCI_MSI */
2846 
2847         adapter->num_rx_queues = 1;
2848         dev_info(&adapter->netdev->dev,
2849                  "Using INTx interrupt, #Rx queues: 1.\n");
2850         adapter->intr.type = VMXNET3_IT_INTX;
2851 
2852         /* INT-X related setting */
2853         adapter->intr.num_intrs = 1;
2854 }
2855 
2856 
2857 static void
2858 vmxnet3_free_intr_resources(struct vmxnet3_adapter *adapter)
2859 {
2860         if (adapter->intr.type == VMXNET3_IT_MSIX)
2861                 pci_disable_msix(adapter->pdev);
2862         else if (adapter->intr.type == VMXNET3_IT_MSI)
2863                 pci_disable_msi(adapter->pdev);
2864         else
2865                 BUG_ON(adapter->intr.type != VMXNET3_IT_INTX);
2866 }
2867 
2868 
2869 static void
2870 vmxnet3_tx_timeout(struct net_device *netdev)
2871 {
2872         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2873         adapter->tx_timeout_count++;
2874 
2875         netdev_err(adapter->netdev, "tx hang\n");
2876         schedule_work(&adapter->work);
2877         netif_wake_queue(adapter->netdev);
2878 }
2879 
2880 
2881 static void
2882 vmxnet3_reset_work(struct work_struct *data)
2883 {
2884         struct vmxnet3_adapter *adapter;
2885 
2886         adapter = container_of(data, struct vmxnet3_adapter, work);
2887 
2888         /* if another thread is resetting the device, no need to proceed */
2889         if (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
2890                 return;
2891 
2892         /* if the device is closed, we must leave it alone */
2893         rtnl_lock();
2894         if (netif_running(adapter->netdev)) {
2895                 netdev_notice(adapter->netdev, "resetting\n");
2896                 vmxnet3_quiesce_dev(adapter);
2897                 vmxnet3_reset_dev(adapter);
2898                 vmxnet3_activate_dev(adapter);
2899         } else {
2900                 netdev_info(adapter->netdev, "already closed\n");
2901         }
2902         rtnl_unlock();
2903 
2904         clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
2905 }
2906 
2907 
2908 static int
2909 vmxnet3_probe_device(struct pci_dev *pdev,
2910                      const struct pci_device_id *id)
2911 {
2912         static const struct net_device_ops vmxnet3_netdev_ops = {
2913                 .ndo_open = vmxnet3_open,
2914                 .ndo_stop = vmxnet3_close,
2915                 .ndo_start_xmit = vmxnet3_xmit_frame,
2916                 .ndo_set_mac_address = vmxnet3_set_mac_addr,
2917                 .ndo_change_mtu = vmxnet3_change_mtu,
2918                 .ndo_set_features = vmxnet3_set_features,
2919                 .ndo_get_stats64 = vmxnet3_get_stats64,
2920                 .ndo_tx_timeout = vmxnet3_tx_timeout,
2921                 .ndo_set_rx_mode = vmxnet3_set_mc,
2922                 .ndo_vlan_rx_add_vid = vmxnet3_vlan_rx_add_vid,
2923                 .ndo_vlan_rx_kill_vid = vmxnet3_vlan_rx_kill_vid,
2924 #ifdef CONFIG_NET_POLL_CONTROLLER
2925                 .ndo_poll_controller = vmxnet3_netpoll,
2926 #endif
2927         };
2928         int err;
2929         bool dma64 = false; /* stupid gcc */
2930         u32 ver;
2931         struct net_device *netdev;
2932         struct vmxnet3_adapter *adapter;
2933         u8 mac[ETH_ALEN];
2934         int size;
2935         int num_tx_queues;
2936         int num_rx_queues;
2937 
2938         if (!pci_msi_enabled())
2939                 enable_mq = 0;
2940 
2941 #ifdef VMXNET3_RSS
2942         if (enable_mq)
2943                 num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES,
2944                                     (int)num_online_cpus());
2945         else
2946 #endif
2947                 num_rx_queues = 1;
2948         num_rx_queues = rounddown_pow_of_two(num_rx_queues);
2949 
2950         if (enable_mq)
2951                 num_tx_queues = min(VMXNET3_DEVICE_MAX_TX_QUEUES,
2952                                     (int)num_online_cpus());
2953         else
2954                 num_tx_queues = 1;
2955 
2956         num_tx_queues = rounddown_pow_of_two(num_tx_queues);
2957         netdev = alloc_etherdev_mq(sizeof(struct vmxnet3_adapter),
2958                                    max(num_tx_queues, num_rx_queues));
2959         dev_info(&pdev->dev,
2960                  "# of Tx queues : %d, # of Rx queues : %d\n",
2961                  num_tx_queues, num_rx_queues);
2962 
2963         if (!netdev)
2964                 return -ENOMEM;
2965 
2966         pci_set_drvdata(pdev, netdev);
2967         adapter = netdev_priv(netdev);
2968         adapter->netdev = netdev;
2969         adapter->pdev = pdev;
2970 
2971         spin_lock_init(&adapter->cmd_lock);
2972         adapter->adapter_pa = dma_map_single(&adapter->pdev->dev, adapter,
2973                                              sizeof(struct vmxnet3_adapter),
2974                                              PCI_DMA_TODEVICE);
2975         adapter->shared = dma_alloc_coherent(
2976                                 &adapter->pdev->dev,
2977                                 sizeof(struct Vmxnet3_DriverShared),
2978                                 &adapter->shared_pa, GFP_KERNEL);
2979         if (!adapter->shared) {
2980                 dev_err(&pdev->dev, "Failed to allocate memory\n");
2981                 err = -ENOMEM;
2982                 goto err_alloc_shared;
2983         }
2984 
2985         adapter->num_rx_queues = num_rx_queues;
2986         adapter->num_tx_queues = num_tx_queues;
2987         adapter->rx_buf_per_pkt = 1;
2988 
2989         size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues;
2990         size += sizeof(struct Vmxnet3_RxQueueDesc) * adapter->num_rx_queues;
2991         adapter->tqd_start = dma_alloc_coherent(&adapter->pdev->dev, size,
2992                                                 &adapter->queue_desc_pa,
2993                                                 GFP_KERNEL);
2994 
2995         if (!adapter->tqd_start) {
2996                 dev_err(&pdev->dev, "Failed to allocate memory\n");
2997                 err = -ENOMEM;
2998                 goto err_alloc_queue_desc;
2999         }
3000         adapter->rqd_start = (struct Vmxnet3_RxQueueDesc *)(adapter->tqd_start +
3001                                                             adapter->num_tx_queues);
3002 
3003         adapter->pm_conf = dma_alloc_coherent(&adapter->pdev->dev,
3004                                               sizeof(struct Vmxnet3_PMConf),
3005                                               &adapter->pm_conf_pa,
3006                                               GFP_KERNEL);
3007         if (adapter->pm_conf == NULL) {
3008                 err = -ENOMEM;
3009                 goto err_alloc_pm;
3010         }
3011 
3012 #ifdef VMXNET3_RSS
3013 
3014         adapter->rss_conf = dma_alloc_coherent(&adapter->pdev->dev,
3015                                                sizeof(struct UPT1_RSSConf),
3016                                                &adapter->rss_conf_pa,
3017                                                GFP_KERNEL);
3018         if (adapter->rss_conf == NULL) {
3019                 err = -ENOMEM;
3020                 goto err_alloc_rss;
3021         }
3022 #endif /* VMXNET3_RSS */
3023 
3024         err = vmxnet3_alloc_pci_resources(adapter, &dma64);
3025         if (err < 0)
3026                 goto err_alloc_pci;
3027 
3028         ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_VRRS);
3029         if (ver & 1) {
3030                 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_VRRS, 1);
3031         } else {
3032                 dev_err(&pdev->dev,
3033                         "Incompatible h/w version (0x%x) for adapter\n", ver);
3034                 err = -EBUSY;
3035                 goto err_ver;
3036         }
3037 
3038         ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_UVRS);
3039         if (ver & 1) {
3040                 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_UVRS, 1);
3041         } else {
3042                 dev_err(&pdev->dev,
3043                         "Incompatible upt version (0x%x) for adapter\n", ver);
3044                 err = -EBUSY;
3045                 goto err_ver;
3046         }
3047 
3048         SET_NETDEV_DEV(netdev, &pdev->dev);
3049         vmxnet3_declare_features(adapter, dma64);
3050 
3051         if (adapter->num_tx_queues == adapter->num_rx_queues)
3052                 adapter->share_intr = VMXNET3_INTR_BUDDYSHARE;
3053         else
3054                 adapter->share_intr = VMXNET3_INTR_DONTSHARE;
3055 
3056         vmxnet3_alloc_intr_resources(adapter);
3057 
3058 #ifdef VMXNET3_RSS
3059         if (adapter->num_rx_queues > 1 &&
3060             adapter->intr.type == VMXNET3_IT_MSIX) {
3061                 adapter->rss = true;
3062                 netdev->hw_features |= NETIF_F_RXHASH;
3063                 netdev->features |= NETIF_F_RXHASH;
3064                 dev_dbg(&pdev->dev, "RSS is enabled.\n");
3065         } else {
3066                 adapter->rss = false;
3067         }
3068 #endif
3069 
3070         vmxnet3_read_mac_addr(adapter, mac);
3071         memcpy(netdev->dev_addr,  mac, netdev->addr_len);
3072 
3073         netdev->netdev_ops = &vmxnet3_netdev_ops;
3074         vmxnet3_set_ethtool_ops(netdev);
3075         netdev->watchdog_timeo = 5 * HZ;
3076 
3077         INIT_WORK(&adapter->work, vmxnet3_reset_work);
3078         set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);
3079 
3080         if (adapter->intr.type == VMXNET3_IT_MSIX) {
3081                 int i;
3082                 for (i = 0; i < adapter->num_rx_queues; i++) {
3083                         netif_napi_add(adapter->netdev,
3084                                        &adapter->rx_queue[i].napi,
3085                                        vmxnet3_poll_rx_only, 64);
3086                 }
3087         } else {
3088                 netif_napi_add(adapter->netdev, &adapter->rx_queue[0].napi,
3089                                vmxnet3_poll, 64);
3090         }
3091 
3092         netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
3093         netif_set_real_num_rx_queues(adapter->netdev, adapter->num_rx_queues);
3094 
3095         netif_carrier_off(netdev);
3096         err = register_netdev(netdev);
3097 
3098         if (err) {
3099                 dev_err(&pdev->dev, "Failed to register adapter\n");
3100                 goto err_register;
3101         }
3102 
3103         vmxnet3_check_link(adapter, false);
3104         return 0;
3105 
3106 err_register:
3107         vmxnet3_free_intr_resources(adapter);
3108 err_ver:
3109         vmxnet3_free_pci_resources(adapter);
3110 err_alloc_pci:
3111 #ifdef VMXNET3_RSS
3112         dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf),
3113                           adapter->rss_conf, adapter->rss_conf_pa);
3114 err_alloc_rss:
3115 #endif
3116         dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf),
3117                           adapter->pm_conf, adapter->pm_conf_pa);
3118 err_alloc_pm:
3119         dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start,
3120                           adapter->queue_desc_pa);
3121 err_alloc_queue_desc:
3122         dma_free_coherent(&adapter->pdev->dev,
3123                           sizeof(struct Vmxnet3_DriverShared),
3124                           adapter->shared, adapter->shared_pa);
3125 err_alloc_shared:
3126         dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa,
3127                          sizeof(struct vmxnet3_adapter), PCI_DMA_TODEVICE);
3128         free_netdev(netdev);
3129         return err;
3130 }
3131 
3132 
3133 static void
3134 vmxnet3_remove_device(struct pci_dev *pdev)
3135 {
3136         struct net_device *netdev = pci_get_drvdata(pdev);
3137         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
3138         int size = 0;
3139         int num_rx_queues;
3140 
3141 #ifdef VMXNET3_RSS
3142         if (enable_mq)
3143                 num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES,
3144                                     (int)num_online_cpus());
3145         else
3146 #endif
3147                 num_rx_queues = 1;
3148         num_rx_queues = rounddown_pow_of_two(num_rx_queues);
3149 
3150         cancel_work_sync(&adapter->work);
3151 
3152         unregister_netdev(netdev);
3153 
3154         vmxnet3_free_intr_resources(adapter);
3155         vmxnet3_free_pci_resources(adapter);
3156 #ifdef VMXNET3_RSS
3157         dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf),
3158                           adapter->rss_conf, adapter->rss_conf_pa);
3159 #endif
3160         dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf),
3161                           adapter->pm_conf, adapter->pm_conf_pa);
3162 
3163         size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues;
3164         size += sizeof(struct Vmxnet3_RxQueueDesc) * num_rx_queues;
3165         dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start,
3166                           adapter->queue_desc_pa);
3167         dma_free_coherent(&adapter->pdev->dev,
3168                           sizeof(struct Vmxnet3_DriverShared),
3169                           adapter->shared, adapter->shared_pa);
3170         dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa,
3171                          sizeof(struct vmxnet3_adapter), PCI_DMA_TODEVICE);
3172         free_netdev(netdev);
3173 }
3174 
3175 
3176 #ifdef CONFIG_PM
3177 
3178 static int
3179 vmxnet3_suspend(struct device *device)
3180 {
3181         struct pci_dev *pdev = to_pci_dev(device);
3182         struct net_device *netdev = pci_get_drvdata(pdev);
3183         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
3184         struct Vmxnet3_PMConf *pmConf;
3185         struct ethhdr *ehdr;
3186         struct arphdr *ahdr;
3187         u8 *arpreq;
3188         struct in_device *in_dev;
3189         struct in_ifaddr *ifa;
3190         unsigned long flags;
3191         int i = 0;
3192 
3193         if (!netif_running(netdev))
3194                 return 0;
3195 
3196         for (i = 0; i < adapter->num_rx_queues; i++)
3197                 napi_disable(&adapter->rx_queue[i].napi);
3198 
3199         vmxnet3_disable_all_intrs(adapter);
3200         vmxnet3_free_irqs(adapter);
3201         vmxnet3_free_intr_resources(adapter);
3202 
3203         netif_device_detach(netdev);
3204         netif_tx_stop_all_queues(netdev);
3205 
3206         /* Create wake-up filters. */
3207         pmConf = adapter->pm_conf;
3208         memset(pmConf, 0, sizeof(*pmConf));
3209 
3210         if (adapter->wol & WAKE_UCAST) {
3211                 pmConf->filters[i].patternSize = ETH_ALEN;
3212                 pmConf->filters[i].maskSize = 1;
3213                 memcpy(pmConf->filters[i].pattern, netdev->dev_addr, ETH_ALEN);
3214                 pmConf->filters[i].mask[0] = 0x3F; /* LSB ETH_ALEN bits */
3215 
3216                 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER;
3217                 i++;
3218         }
3219 
3220         if (adapter->wol & WAKE_ARP) {
3221                 in_dev = in_dev_get(netdev);
3222                 if (!in_dev)
3223                         goto skip_arp;
3224 
3225                 ifa = (struct in_ifaddr *)in_dev->ifa_list;
3226                 if (!ifa)
3227                         goto skip_arp;
3228 
3229                 pmConf->filters[i].patternSize = ETH_HLEN + /* Ethernet header*/
3230                         sizeof(struct arphdr) +         /* ARP header */
3231                         2 * ETH_ALEN +          /* 2 Ethernet addresses*/
3232                         2 * sizeof(u32);        /*2 IPv4 addresses */
3233                 pmConf->filters[i].maskSize =
3234                         (pmConf->filters[i].patternSize - 1) / 8 + 1;
3235 
3236                 /* ETH_P_ARP in Ethernet header. */
3237                 ehdr = (struct ethhdr *)pmConf->filters[i].pattern;
3238                 ehdr->h_proto = htons(ETH_P_ARP);
3239 
3240                 /* ARPOP_REQUEST in ARP header. */
3241                 ahdr = (struct arphdr *)&pmConf->filters[i].pattern[ETH_HLEN];
3242                 ahdr->ar_op = htons(ARPOP_REQUEST);
3243                 arpreq = (u8 *)(ahdr + 1);
3244 
3245                 /* The Unicast IPv4 address in 'tip' field. */
3246                 arpreq += 2 * ETH_ALEN + sizeof(u32);
3247                 *(u32 *)arpreq = ifa->ifa_address;
3248 
3249                 /* The mask for the relevant bits. */
3250                 pmConf->filters[i].mask[0] = 0x00;
3251                 pmConf->filters[i].mask[1] = 0x30; /* ETH_P_ARP */
3252                 pmConf->filters[i].mask[2] = 0x30; /* ARPOP_REQUEST */
3253                 pmConf->filters[i].mask[3] = 0x00;
3254                 pmConf->filters[i].mask[4] = 0xC0; /* IPv4 TIP */
3255                 pmConf->filters[i].mask[5] = 0x03; /* IPv4 TIP */
3256                 in_dev_put(in_dev);
3257 
3258                 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER;
3259                 i++;
3260         }
3261 
3262 skip_arp:
3263         if (adapter->wol & WAKE_MAGIC)
3264                 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_MAGIC;
3265 
3266         pmConf->numFilters = i;
3267 
3268         adapter->shared->devRead.pmConfDesc.confVer = cpu_to_le32(1);
3269         adapter->shared->devRead.pmConfDesc.confLen = cpu_to_le32(sizeof(
3270                                                                   *pmConf));
3271         adapter->shared->devRead.pmConfDesc.confPA =
3272                 cpu_to_le64(adapter->pm_conf_pa);
3273 
3274         spin_lock_irqsave(&adapter->cmd_lock, flags);
3275         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
3276                                VMXNET3_CMD_UPDATE_PMCFG);
3277         spin_unlock_irqrestore(&adapter->cmd_lock, flags);
3278 
3279         pci_save_state(pdev);
3280         pci_enable_wake(pdev, pci_choose_state(pdev, PMSG_SUSPEND),
3281                         adapter->wol);
3282         pci_disable_device(pdev);
3283         pci_set_power_state(pdev, pci_choose_state(pdev, PMSG_SUSPEND));
3284 
3285         return 0;
3286 }
3287 
3288 
3289 static int
3290 vmxnet3_resume(struct device *device)
3291 {
3292         int err, i = 0;
3293         unsigned long flags;
3294         struct pci_dev *pdev = to_pci_dev(device);
3295         struct net_device *netdev = pci_get_drvdata(pdev);
3296         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
3297         struct Vmxnet3_PMConf *pmConf;
3298 
3299         if (!netif_running(netdev))
3300                 return 0;
3301 
3302         /* Destroy wake-up filters. */
3303         pmConf = adapter->pm_conf;
3304         memset(pmConf, 0, sizeof(*pmConf));
3305 
3306         adapter->shared->devRead.pmConfDesc.confVer = cpu_to_le32(1);
3307         adapter->shared->devRead.pmConfDesc.confLen = cpu_to_le32(sizeof(
3308                                                                   *pmConf));
3309         adapter->shared->devRead.pmConfDesc.confPA =
3310                 cpu_to_le64(adapter->pm_conf_pa);
3311 
3312         netif_device_attach(netdev);
3313         pci_set_power_state(pdev, PCI_D0);
3314         pci_restore_state(pdev);
3315         err = pci_enable_device_mem(pdev);
3316         if (err != 0)
3317                 return err;
3318 
3319         pci_enable_wake(pdev, PCI_D0, 0);
3320 
3321         spin_lock_irqsave(&adapter->cmd_lock, flags);
3322         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
3323                                VMXNET3_CMD_UPDATE_PMCFG);
3324         spin_unlock_irqrestore(&adapter->cmd_lock, flags);
3325         vmxnet3_alloc_intr_resources(adapter);
3326         vmxnet3_request_irqs(adapter);
3327         for (i = 0; i < adapter->num_rx_queues; i++)
3328                 napi_enable(&adapter->rx_queue[i].napi);
3329         vmxnet3_enable_all_intrs(adapter);
3330 
3331         return 0;
3332 }
3333 
3334 static const struct dev_pm_ops vmxnet3_pm_ops = {
3335         .suspend = vmxnet3_suspend,
3336         .resume = vmxnet3_resume,
3337 };
3338 #endif
3339 
3340 static struct pci_driver vmxnet3_driver = {
3341         .name           = vmxnet3_driver_name,
3342         .id_table       = vmxnet3_pciid_table,
3343         .probe          = vmxnet3_probe_device,
3344         .remove         = vmxnet3_remove_device,
3345 #ifdef CONFIG_PM
3346         .driver.pm      = &vmxnet3_pm_ops,
3347 #endif
3348 };
3349 
3350 
3351 static int __init
3352 vmxnet3_init_module(void)
3353 {
3354         pr_info("%s - version %s\n", VMXNET3_DRIVER_DESC,
3355                 VMXNET3_DRIVER_VERSION_REPORT);
3356         return pci_register_driver(&vmxnet3_driver);
3357 }
3358 
3359 module_init(vmxnet3_init_module);
3360 
3361 
3362 static void
3363 vmxnet3_exit_module(void)
3364 {
3365         pci_unregister_driver(&vmxnet3_driver);
3366 }
3367 
3368 module_exit(vmxnet3_exit_module);
3369 
3370 MODULE_AUTHOR("VMware, Inc.");
3371 MODULE_DESCRIPTION(VMXNET3_DRIVER_DESC);
3372 MODULE_LICENSE("GPL v2");
3373 MODULE_VERSION(VMXNET3_DRIVER_VERSION_STRING);
3374 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us