Version:  2.0.40 2.2.26 2.4.37 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1 4.2

Linux/drivers/infiniband/hw/ipath/ipath_driver.c

  1 /*
  2  * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved.
  3  * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
  4  *
  5  * This software is available to you under a choice of one of two
  6  * licenses.  You may choose to be licensed under the terms of the GNU
  7  * General Public License (GPL) Version 2, available from the file
  8  * COPYING in the main directory of this source tree, or the
  9  * OpenIB.org BSD license below:
 10  *
 11  *     Redistribution and use in source and binary forms, with or
 12  *     without modification, are permitted provided that the following
 13  *     conditions are met:
 14  *
 15  *      - Redistributions of source code must retain the above
 16  *        copyright notice, this list of conditions and the following
 17  *        disclaimer.
 18  *
 19  *      - Redistributions in binary form must reproduce the above
 20  *        copyright notice, this list of conditions and the following
 21  *        disclaimer in the documentation and/or other materials
 22  *        provided with the distribution.
 23  *
 24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 31  * SOFTWARE.
 32  */
 33 
 34 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 35 
 36 #include <linux/sched.h>
 37 #include <linux/spinlock.h>
 38 #include <linux/idr.h>
 39 #include <linux/pci.h>
 40 #include <linux/io.h>
 41 #include <linux/delay.h>
 42 #include <linux/netdevice.h>
 43 #include <linux/vmalloc.h>
 44 #include <linux/bitmap.h>
 45 #include <linux/slab.h>
 46 #include <linux/module.h>
 47 #ifdef CONFIG_X86_64
 48 #include <asm/pat.h>
 49 #endif
 50 
 51 #include "ipath_kernel.h"
 52 #include "ipath_verbs.h"
 53 
 54 static void ipath_update_pio_bufs(struct ipath_devdata *);
 55 
 56 const char *ipath_get_unit_name(int unit)
 57 {
 58         static char iname[16];
 59         snprintf(iname, sizeof iname, "infinipath%u", unit);
 60         return iname;
 61 }
 62 
 63 #define DRIVER_LOAD_MSG "QLogic " IPATH_DRV_NAME " loaded: "
 64 #define PFX IPATH_DRV_NAME ": "
 65 
 66 /*
 67  * The size has to be longer than this string, so we can append
 68  * board/chip information to it in the init code.
 69  */
 70 const char ib_ipath_version[] = IPATH_IDSTR "\n";
 71 
 72 static struct idr unit_table;
 73 DEFINE_SPINLOCK(ipath_devs_lock);
 74 LIST_HEAD(ipath_dev_list);
 75 
 76 wait_queue_head_t ipath_state_wait;
 77 
 78 unsigned ipath_debug = __IPATH_INFO;
 79 
 80 module_param_named(debug, ipath_debug, uint, S_IWUSR | S_IRUGO);
 81 MODULE_PARM_DESC(debug, "mask for debug prints");
 82 EXPORT_SYMBOL_GPL(ipath_debug);
 83 
 84 unsigned ipath_mtu4096 = 1; /* max 4KB IB mtu by default, if supported */
 85 module_param_named(mtu4096, ipath_mtu4096, uint, S_IRUGO);
 86 MODULE_PARM_DESC(mtu4096, "enable MTU of 4096 bytes, if supported");
 87 
 88 static unsigned ipath_hol_timeout_ms = 13000;
 89 module_param_named(hol_timeout_ms, ipath_hol_timeout_ms, uint, S_IRUGO);
 90 MODULE_PARM_DESC(hol_timeout_ms,
 91         "duration of user app suspension after link failure");
 92 
 93 unsigned ipath_linkrecovery = 1;
 94 module_param_named(linkrecovery, ipath_linkrecovery, uint, S_IWUSR | S_IRUGO);
 95 MODULE_PARM_DESC(linkrecovery, "enable workaround for link recovery issue");
 96 
 97 MODULE_LICENSE("GPL");
 98 MODULE_AUTHOR("QLogic <support@qlogic.com>");
 99 MODULE_DESCRIPTION("QLogic InfiniPath driver");
100 
101 /*
102  * Table to translate the LINKTRAININGSTATE portion of
103  * IBCStatus to a human-readable form.
104  */
105 const char *ipath_ibcstatus_str[] = {
106         "Disabled",
107         "LinkUp",
108         "PollActive",
109         "PollQuiet",
110         "SleepDelay",
111         "SleepQuiet",
112         "LState6",              /* unused */
113         "LState7",              /* unused */
114         "CfgDebounce",
115         "CfgRcvfCfg",
116         "CfgWaitRmt",
117         "CfgIdle",
118         "RecovRetrain",
119         "CfgTxRevLane",         /* unused before IBA7220 */
120         "RecovWaitRmt",
121         "RecovIdle",
122         /* below were added for IBA7220 */
123         "CfgEnhanced",
124         "CfgTest",
125         "CfgWaitRmtTest",
126         "CfgWaitCfgEnhanced",
127         "SendTS_T",
128         "SendTstIdles",
129         "RcvTS_T",
130         "SendTst_TS1s",
131         "LTState18", "LTState19", "LTState1A", "LTState1B",
132         "LTState1C", "LTState1D", "LTState1E", "LTState1F"
133 };
134 
135 static void ipath_remove_one(struct pci_dev *);
136 static int ipath_init_one(struct pci_dev *, const struct pci_device_id *);
137 
138 /* Only needed for registration, nothing else needs this info */
139 #define PCI_VENDOR_ID_PATHSCALE 0x1fc1
140 #define PCI_DEVICE_ID_INFINIPATH_HT 0xd
141 
142 /* Number of seconds before our card status check...  */
143 #define STATUS_TIMEOUT 60
144 
145 static const struct pci_device_id ipath_pci_tbl[] = {
146         { PCI_DEVICE(PCI_VENDOR_ID_PATHSCALE, PCI_DEVICE_ID_INFINIPATH_HT) },
147         { 0, }
148 };
149 
150 MODULE_DEVICE_TABLE(pci, ipath_pci_tbl);
151 
152 static struct pci_driver ipath_driver = {
153         .name = IPATH_DRV_NAME,
154         .probe = ipath_init_one,
155         .remove = ipath_remove_one,
156         .id_table = ipath_pci_tbl,
157         .driver = {
158                 .groups = ipath_driver_attr_groups,
159         },
160 };
161 
162 static inline void read_bars(struct ipath_devdata *dd, struct pci_dev *dev,
163                              u32 *bar0, u32 *bar1)
164 {
165         int ret;
166 
167         ret = pci_read_config_dword(dev, PCI_BASE_ADDRESS_0, bar0);
168         if (ret)
169                 ipath_dev_err(dd, "failed to read bar0 before enable: "
170                               "error %d\n", -ret);
171 
172         ret = pci_read_config_dword(dev, PCI_BASE_ADDRESS_1, bar1);
173         if (ret)
174                 ipath_dev_err(dd, "failed to read bar1 before enable: "
175                               "error %d\n", -ret);
176 
177         ipath_dbg("Read bar0 %x bar1 %x\n", *bar0, *bar1);
178 }
179 
180 static void ipath_free_devdata(struct pci_dev *pdev,
181                                struct ipath_devdata *dd)
182 {
183         unsigned long flags;
184 
185         pci_set_drvdata(pdev, NULL);
186 
187         if (dd->ipath_unit != -1) {
188                 spin_lock_irqsave(&ipath_devs_lock, flags);
189                 idr_remove(&unit_table, dd->ipath_unit);
190                 list_del(&dd->ipath_list);
191                 spin_unlock_irqrestore(&ipath_devs_lock, flags);
192         }
193         vfree(dd);
194 }
195 
196 static struct ipath_devdata *ipath_alloc_devdata(struct pci_dev *pdev)
197 {
198         unsigned long flags;
199         struct ipath_devdata *dd;
200         int ret;
201 
202         dd = vzalloc(sizeof(*dd));
203         if (!dd) {
204                 dd = ERR_PTR(-ENOMEM);
205                 goto bail;
206         }
207         dd->ipath_unit = -1;
208 
209         idr_preload(GFP_KERNEL);
210         spin_lock_irqsave(&ipath_devs_lock, flags);
211 
212         ret = idr_alloc(&unit_table, dd, 0, 0, GFP_NOWAIT);
213         if (ret < 0) {
214                 printk(KERN_ERR IPATH_DRV_NAME
215                        ": Could not allocate unit ID: error %d\n", -ret);
216                 ipath_free_devdata(pdev, dd);
217                 dd = ERR_PTR(ret);
218                 goto bail_unlock;
219         }
220         dd->ipath_unit = ret;
221 
222         dd->pcidev = pdev;
223         pci_set_drvdata(pdev, dd);
224 
225         list_add(&dd->ipath_list, &ipath_dev_list);
226 
227 bail_unlock:
228         spin_unlock_irqrestore(&ipath_devs_lock, flags);
229         idr_preload_end();
230 bail:
231         return dd;
232 }
233 
234 static inline struct ipath_devdata *__ipath_lookup(int unit)
235 {
236         return idr_find(&unit_table, unit);
237 }
238 
239 struct ipath_devdata *ipath_lookup(int unit)
240 {
241         struct ipath_devdata *dd;
242         unsigned long flags;
243 
244         spin_lock_irqsave(&ipath_devs_lock, flags);
245         dd = __ipath_lookup(unit);
246         spin_unlock_irqrestore(&ipath_devs_lock, flags);
247 
248         return dd;
249 }
250 
251 int ipath_count_units(int *npresentp, int *nupp, int *maxportsp)
252 {
253         int nunits, npresent, nup;
254         struct ipath_devdata *dd;
255         unsigned long flags;
256         int maxports;
257 
258         nunits = npresent = nup = maxports = 0;
259 
260         spin_lock_irqsave(&ipath_devs_lock, flags);
261 
262         list_for_each_entry(dd, &ipath_dev_list, ipath_list) {
263                 nunits++;
264                 if ((dd->ipath_flags & IPATH_PRESENT) && dd->ipath_kregbase)
265                         npresent++;
266                 if (dd->ipath_lid &&
267                     !(dd->ipath_flags & (IPATH_DISABLED | IPATH_LINKDOWN
268                                          | IPATH_LINKUNK)))
269                         nup++;
270                 if (dd->ipath_cfgports > maxports)
271                         maxports = dd->ipath_cfgports;
272         }
273 
274         spin_unlock_irqrestore(&ipath_devs_lock, flags);
275 
276         if (npresentp)
277                 *npresentp = npresent;
278         if (nupp)
279                 *nupp = nup;
280         if (maxportsp)
281                 *maxportsp = maxports;
282 
283         return nunits;
284 }
285 
286 /*
287  * These next two routines are placeholders in case we don't have per-arch
288  * code for controlling write combining.  If explicit control of write
289  * combining is not available, performance will probably be awful.
290  */
291 
292 int __attribute__((weak)) ipath_enable_wc(struct ipath_devdata *dd)
293 {
294         return -EOPNOTSUPP;
295 }
296 
297 void __attribute__((weak)) ipath_disable_wc(struct ipath_devdata *dd)
298 {
299 }
300 
301 /*
302  * Perform a PIO buffer bandwidth write test, to verify proper system
303  * configuration.  Even when all the setup calls work, occasionally
304  * BIOS or other issues can prevent write combining from working, or
305  * can cause other bandwidth problems to the chip.
306  *
307  * This test simply writes the same buffer over and over again, and
308  * measures close to the peak bandwidth to the chip (not testing
309  * data bandwidth to the wire).   On chips that use an address-based
310  * trigger to send packets to the wire, this is easy.  On chips that
311  * use a count to trigger, we want to make sure that the packet doesn't
312  * go out on the wire, or trigger flow control checks.
313  */
314 static void ipath_verify_pioperf(struct ipath_devdata *dd)
315 {
316         u32 pbnum, cnt, lcnt;
317         u32 __iomem *piobuf;
318         u32 *addr;
319         u64 msecs, emsecs;
320 
321         piobuf = ipath_getpiobuf(dd, 0, &pbnum);
322         if (!piobuf) {
323                 dev_info(&dd->pcidev->dev,
324                         "No PIObufs for checking perf, skipping\n");
325                 return;
326         }
327 
328         /*
329          * Enough to give us a reasonable test, less than piobuf size, and
330          * likely multiple of store buffer length.
331          */
332         cnt = 1024;
333 
334         addr = vmalloc(cnt);
335         if (!addr) {
336                 dev_info(&dd->pcidev->dev,
337                         "Couldn't get memory for checking PIO perf,"
338                         " skipping\n");
339                 goto done;
340         }
341 
342         preempt_disable();  /* we want reasonably accurate elapsed time */
343         msecs = 1 + jiffies_to_msecs(jiffies);
344         for (lcnt = 0; lcnt < 10000U; lcnt++) {
345                 /* wait until we cross msec boundary */
346                 if (jiffies_to_msecs(jiffies) >= msecs)
347                         break;
348                 udelay(1);
349         }
350 
351         ipath_disable_armlaunch(dd);
352 
353         /*
354          * length 0, no dwords actually sent, and mark as VL15
355          * on chips where that may matter (due to IB flowcontrol)
356          */
357         if ((dd->ipath_flags & IPATH_HAS_PBC_CNT))
358                 writeq(1UL << 63, piobuf);
359         else
360                 writeq(0, piobuf);
361         ipath_flush_wc();
362 
363         /*
364          * this is only roughly accurate, since even with preempt we
365          * still take interrupts that could take a while.   Running for
366          * >= 5 msec seems to get us "close enough" to accurate values
367          */
368         msecs = jiffies_to_msecs(jiffies);
369         for (emsecs = lcnt = 0; emsecs <= 5UL; lcnt++) {
370                 __iowrite32_copy(piobuf + 64, addr, cnt >> 2);
371                 emsecs = jiffies_to_msecs(jiffies) - msecs;
372         }
373 
374         /* 1 GiB/sec, slightly over IB SDR line rate */
375         if (lcnt < (emsecs * 1024U))
376                 ipath_dev_err(dd,
377                         "Performance problem: bandwidth to PIO buffers is "
378                         "only %u MiB/sec\n",
379                         lcnt / (u32) emsecs);
380         else
381                 ipath_dbg("PIO buffer bandwidth %u MiB/sec is OK\n",
382                         lcnt / (u32) emsecs);
383 
384         preempt_enable();
385 
386         vfree(addr);
387 
388 done:
389         /* disarm piobuf, so it's available again */
390         ipath_disarm_piobufs(dd, pbnum, 1);
391         ipath_enable_armlaunch(dd);
392 }
393 
394 static void cleanup_device(struct ipath_devdata *dd);
395 
396 static int ipath_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
397 {
398         int ret, len, j;
399         struct ipath_devdata *dd;
400         unsigned long long addr;
401         u32 bar0 = 0, bar1 = 0;
402 
403 #ifdef CONFIG_X86_64
404         if (pat_enabled()) {
405                 pr_warn("ipath needs PAT disabled, boot with nopat kernel parameter\n");
406                 ret = -ENODEV;
407                 goto bail;
408         }
409 #endif
410 
411         dd = ipath_alloc_devdata(pdev);
412         if (IS_ERR(dd)) {
413                 ret = PTR_ERR(dd);
414                 printk(KERN_ERR IPATH_DRV_NAME
415                        ": Could not allocate devdata: error %d\n", -ret);
416                 goto bail;
417         }
418 
419         ipath_cdbg(VERBOSE, "initializing unit #%u\n", dd->ipath_unit);
420 
421         ret = pci_enable_device(pdev);
422         if (ret) {
423                 /* This can happen iff:
424                  *
425                  * We did a chip reset, and then failed to reprogram the
426                  * BAR, or the chip reset due to an internal error.  We then
427                  * unloaded the driver and reloaded it.
428                  *
429                  * Both reset cases set the BAR back to initial state.  For
430                  * the latter case, the AER sticky error bit at offset 0x718
431                  * should be set, but the Linux kernel doesn't yet know
432                  * about that, it appears.  If the original BAR was retained
433                  * in the kernel data structures, this may be OK.
434                  */
435                 ipath_dev_err(dd, "enable unit %d failed: error %d\n",
436                               dd->ipath_unit, -ret);
437                 goto bail_devdata;
438         }
439         addr = pci_resource_start(pdev, 0);
440         len = pci_resource_len(pdev, 0);
441         ipath_cdbg(VERBOSE, "regbase (0) %llx len %d irq %d, vend %x/%x "
442                    "driver_data %lx\n", addr, len, pdev->irq, ent->vendor,
443                    ent->device, ent->driver_data);
444 
445         read_bars(dd, pdev, &bar0, &bar1);
446 
447         if (!bar1 && !(bar0 & ~0xf)) {
448                 if (addr) {
449                         dev_info(&pdev->dev, "BAR is 0 (probable RESET), "
450                                  "rewriting as %llx\n", addr);
451                         ret = pci_write_config_dword(
452                                 pdev, PCI_BASE_ADDRESS_0, addr);
453                         if (ret) {
454                                 ipath_dev_err(dd, "rewrite of BAR0 "
455                                               "failed: err %d\n", -ret);
456                                 goto bail_disable;
457                         }
458                         ret = pci_write_config_dword(
459                                 pdev, PCI_BASE_ADDRESS_1, addr >> 32);
460                         if (ret) {
461                                 ipath_dev_err(dd, "rewrite of BAR1 "
462                                               "failed: err %d\n", -ret);
463                                 goto bail_disable;
464                         }
465                 } else {
466                         ipath_dev_err(dd, "BAR is 0 (probable RESET), "
467                                       "not usable until reboot\n");
468                         ret = -ENODEV;
469                         goto bail_disable;
470                 }
471         }
472 
473         ret = pci_request_regions(pdev, IPATH_DRV_NAME);
474         if (ret) {
475                 dev_info(&pdev->dev, "pci_request_regions unit %u fails: "
476                          "err %d\n", dd->ipath_unit, -ret);
477                 goto bail_disable;
478         }
479 
480         ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
481         if (ret) {
482                 /*
483                  * if the 64 bit setup fails, try 32 bit.  Some systems
484                  * do not setup 64 bit maps on systems with 2GB or less
485                  * memory installed.
486                  */
487                 ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
488                 if (ret) {
489                         dev_info(&pdev->dev,
490                                 "Unable to set DMA mask for unit %u: %d\n",
491                                 dd->ipath_unit, ret);
492                         goto bail_regions;
493                 }
494                 else {
495                         ipath_dbg("No 64bit DMA mask, used 32 bit mask\n");
496                         ret = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
497                         if (ret)
498                                 dev_info(&pdev->dev,
499                                         "Unable to set DMA consistent mask "
500                                         "for unit %u: %d\n",
501                                         dd->ipath_unit, ret);
502 
503                 }
504         }
505         else {
506                 ret = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
507                 if (ret)
508                         dev_info(&pdev->dev,
509                                 "Unable to set DMA consistent mask "
510                                 "for unit %u: %d\n",
511                                 dd->ipath_unit, ret);
512         }
513 
514         pci_set_master(pdev);
515 
516         /*
517          * Save BARs to rewrite after device reset.  Save all 64 bits of
518          * BAR, just in case.
519          */
520         dd->ipath_pcibar0 = addr;
521         dd->ipath_pcibar1 = addr >> 32;
522         dd->ipath_deviceid = ent->device;       /* save for later use */
523         dd->ipath_vendorid = ent->vendor;
524 
525         /* setup the chip-specific functions, as early as possible. */
526         switch (ent->device) {
527         case PCI_DEVICE_ID_INFINIPATH_HT:
528                 ipath_init_iba6110_funcs(dd);
529                 break;
530 
531         default:
532                 ipath_dev_err(dd, "Found unknown QLogic deviceid 0x%x, "
533                               "failing\n", ent->device);
534                 return -ENODEV;
535         }
536 
537         for (j = 0; j < 6; j++) {
538                 if (!pdev->resource[j].start)
539                         continue;
540                 ipath_cdbg(VERBOSE, "BAR %d %pR, len %llx\n",
541                            j, &pdev->resource[j],
542                            (unsigned long long)pci_resource_len(pdev, j));
543         }
544 
545         if (!addr) {
546                 ipath_dev_err(dd, "No valid address in BAR 0!\n");
547                 ret = -ENODEV;
548                 goto bail_regions;
549         }
550 
551         dd->ipath_pcirev = pdev->revision;
552 
553 #if defined(__powerpc__)
554         /* There isn't a generic way to specify writethrough mappings */
555         dd->ipath_kregbase = __ioremap(addr, len,
556                 (_PAGE_NO_CACHE|_PAGE_WRITETHRU));
557 #else
558         /* XXX: split this properly to enable on PAT */
559         dd->ipath_kregbase = ioremap_nocache(addr, len);
560 #endif
561 
562         if (!dd->ipath_kregbase) {
563                 ipath_dbg("Unable to map io addr %llx to kvirt, failing\n",
564                           addr);
565                 ret = -ENOMEM;
566                 goto bail_iounmap;
567         }
568         dd->ipath_kregend = (u64 __iomem *)
569                 ((void __iomem *)dd->ipath_kregbase + len);
570         dd->ipath_physaddr = addr;      /* used for io_remap, etc. */
571         /* for user mmap */
572         ipath_cdbg(VERBOSE, "mapped io addr %llx to kregbase %p\n",
573                    addr, dd->ipath_kregbase);
574 
575         if (dd->ipath_f_bus(dd, pdev))
576                 ipath_dev_err(dd, "Failed to setup config space; "
577                               "continuing anyway\n");
578 
579         /*
580          * set up our interrupt handler; IRQF_SHARED probably not needed,
581          * since MSI interrupts shouldn't be shared but won't  hurt for now.
582          * check 0 irq after we return from chip-specific bus setup, since
583          * that can affect this due to setup
584          */
585         if (!dd->ipath_irq)
586                 ipath_dev_err(dd, "irq is 0, BIOS error?  Interrupts won't "
587                               "work\n");
588         else {
589                 ret = request_irq(dd->ipath_irq, ipath_intr, IRQF_SHARED,
590                                   IPATH_DRV_NAME, dd);
591                 if (ret) {
592                         ipath_dev_err(dd, "Couldn't setup irq handler, "
593                                       "irq=%d: %d\n", dd->ipath_irq, ret);
594                         goto bail_iounmap;
595                 }
596         }
597 
598         ret = ipath_init_chip(dd, 0);   /* do the chip-specific init */
599         if (ret)
600                 goto bail_irqsetup;
601 
602         ret = ipath_enable_wc(dd);
603 
604         if (ret)
605                 ret = 0;
606 
607         ipath_verify_pioperf(dd);
608 
609         ipath_device_create_group(&pdev->dev, dd);
610         ipathfs_add_device(dd);
611         ipath_user_add(dd);
612         ipath_diag_add(dd);
613         ipath_register_ib_device(dd);
614 
615         goto bail;
616 
617 bail_irqsetup:
618         cleanup_device(dd);
619 
620         if (dd->ipath_irq)
621                 dd->ipath_f_free_irq(dd);
622 
623         if (dd->ipath_f_cleanup)
624                 dd->ipath_f_cleanup(dd);
625 
626 bail_iounmap:
627         iounmap((volatile void __iomem *) dd->ipath_kregbase);
628 
629 bail_regions:
630         pci_release_regions(pdev);
631 
632 bail_disable:
633         pci_disable_device(pdev);
634 
635 bail_devdata:
636         ipath_free_devdata(pdev, dd);
637 
638 bail:
639         return ret;
640 }
641 
642 static void cleanup_device(struct ipath_devdata *dd)
643 {
644         int port;
645         struct ipath_portdata **tmp;
646         unsigned long flags;
647 
648         if (*dd->ipath_statusp & IPATH_STATUS_CHIP_PRESENT) {
649                 /* can't do anything more with chip; needs re-init */
650                 *dd->ipath_statusp &= ~IPATH_STATUS_CHIP_PRESENT;
651                 if (dd->ipath_kregbase) {
652                         /*
653                          * if we haven't already cleaned up before these are
654                          * to ensure any register reads/writes "fail" until
655                          * re-init
656                          */
657                         dd->ipath_kregbase = NULL;
658                         dd->ipath_uregbase = 0;
659                         dd->ipath_sregbase = 0;
660                         dd->ipath_cregbase = 0;
661                         dd->ipath_kregsize = 0;
662                 }
663                 ipath_disable_wc(dd);
664         }
665 
666         if (dd->ipath_spectriggerhit)
667                 dev_info(&dd->pcidev->dev, "%lu special trigger hits\n",
668                          dd->ipath_spectriggerhit);
669 
670         if (dd->ipath_pioavailregs_dma) {
671                 dma_free_coherent(&dd->pcidev->dev, PAGE_SIZE,
672                                   (void *) dd->ipath_pioavailregs_dma,
673                                   dd->ipath_pioavailregs_phys);
674                 dd->ipath_pioavailregs_dma = NULL;
675         }
676         if (dd->ipath_dummy_hdrq) {
677                 dma_free_coherent(&dd->pcidev->dev,
678                         dd->ipath_pd[0]->port_rcvhdrq_size,
679                         dd->ipath_dummy_hdrq, dd->ipath_dummy_hdrq_phys);
680                 dd->ipath_dummy_hdrq = NULL;
681         }
682 
683         if (dd->ipath_pageshadow) {
684                 struct page **tmpp = dd->ipath_pageshadow;
685                 dma_addr_t *tmpd = dd->ipath_physshadow;
686                 int i, cnt = 0;
687 
688                 ipath_cdbg(VERBOSE, "Unlocking any expTID pages still "
689                            "locked\n");
690                 for (port = 0; port < dd->ipath_cfgports; port++) {
691                         int port_tidbase = port * dd->ipath_rcvtidcnt;
692                         int maxtid = port_tidbase + dd->ipath_rcvtidcnt;
693                         for (i = port_tidbase; i < maxtid; i++) {
694                                 if (!tmpp[i])
695                                         continue;
696                                 pci_unmap_page(dd->pcidev, tmpd[i],
697                                         PAGE_SIZE, PCI_DMA_FROMDEVICE);
698                                 ipath_release_user_pages(&tmpp[i], 1);
699                                 tmpp[i] = NULL;
700                                 cnt++;
701                         }
702                 }
703                 if (cnt) {
704                         ipath_stats.sps_pageunlocks += cnt;
705                         ipath_cdbg(VERBOSE, "There were still %u expTID "
706                                    "entries locked\n", cnt);
707                 }
708                 if (ipath_stats.sps_pagelocks ||
709                     ipath_stats.sps_pageunlocks)
710                         ipath_cdbg(VERBOSE, "%llu pages locked, %llu "
711                                    "unlocked via ipath_m{un}lock\n",
712                                    (unsigned long long)
713                                    ipath_stats.sps_pagelocks,
714                                    (unsigned long long)
715                                    ipath_stats.sps_pageunlocks);
716 
717                 ipath_cdbg(VERBOSE, "Free shadow page tid array at %p\n",
718                            dd->ipath_pageshadow);
719                 tmpp = dd->ipath_pageshadow;
720                 dd->ipath_pageshadow = NULL;
721                 vfree(tmpp);
722 
723                 dd->ipath_egrtidbase = NULL;
724         }
725 
726         /*
727          * free any resources still in use (usually just kernel ports)
728          * at unload; we do for portcnt, because that's what we allocate.
729          * We acquire lock to be really paranoid that ipath_pd isn't being
730          * accessed from some interrupt-related code (that should not happen,
731          * but best to be sure).
732          */
733         spin_lock_irqsave(&dd->ipath_uctxt_lock, flags);
734         tmp = dd->ipath_pd;
735         dd->ipath_pd = NULL;
736         spin_unlock_irqrestore(&dd->ipath_uctxt_lock, flags);
737         for (port = 0; port < dd->ipath_portcnt; port++) {
738                 struct ipath_portdata *pd = tmp[port];
739                 tmp[port] = NULL; /* debugging paranoia */
740                 ipath_free_pddata(dd, pd);
741         }
742         kfree(tmp);
743 }
744 
745 static void ipath_remove_one(struct pci_dev *pdev)
746 {
747         struct ipath_devdata *dd = pci_get_drvdata(pdev);
748 
749         ipath_cdbg(VERBOSE, "removing, pdev=%p, dd=%p\n", pdev, dd);
750 
751         /*
752          * disable the IB link early, to be sure no new packets arrive, which
753          * complicates the shutdown process
754          */
755         ipath_shutdown_device(dd);
756 
757         flush_workqueue(ib_wq);
758 
759         if (dd->verbs_dev)
760                 ipath_unregister_ib_device(dd->verbs_dev);
761 
762         ipath_diag_remove(dd);
763         ipath_user_remove(dd);
764         ipathfs_remove_device(dd);
765         ipath_device_remove_group(&pdev->dev, dd);
766 
767         ipath_cdbg(VERBOSE, "Releasing pci memory regions, dd %p, "
768                    "unit %u\n", dd, (u32) dd->ipath_unit);
769 
770         cleanup_device(dd);
771 
772         /*
773          * turn off rcv, send, and interrupts for all ports, all drivers
774          * should also hard reset the chip here?
775          * free up port 0 (kernel) rcvhdr, egr bufs, and eventually tid bufs
776          * for all versions of the driver, if they were allocated
777          */
778         if (dd->ipath_irq) {
779                 ipath_cdbg(VERBOSE, "unit %u free irq %d\n",
780                            dd->ipath_unit, dd->ipath_irq);
781                 dd->ipath_f_free_irq(dd);
782         } else
783                 ipath_dbg("irq is 0, not doing free_irq "
784                           "for unit %u\n", dd->ipath_unit);
785         /*
786          * we check for NULL here, because it's outside
787          * the kregbase check, and we need to call it
788          * after the free_irq.  Thus it's possible that
789          * the function pointers were never initialized.
790          */
791         if (dd->ipath_f_cleanup)
792                 /* clean up chip-specific stuff */
793                 dd->ipath_f_cleanup(dd);
794 
795         ipath_cdbg(VERBOSE, "Unmapping kregbase %p\n", dd->ipath_kregbase);
796         iounmap((volatile void __iomem *) dd->ipath_kregbase);
797         pci_release_regions(pdev);
798         ipath_cdbg(VERBOSE, "calling pci_disable_device\n");
799         pci_disable_device(pdev);
800 
801         ipath_free_devdata(pdev, dd);
802 }
803 
804 /* general driver use */
805 DEFINE_MUTEX(ipath_mutex);
806 
807 static DEFINE_SPINLOCK(ipath_pioavail_lock);
808 
809 /**
810  * ipath_disarm_piobufs - cancel a range of PIO buffers
811  * @dd: the infinipath device
812  * @first: the first PIO buffer to cancel
813  * @cnt: the number of PIO buffers to cancel
814  *
815  * cancel a range of PIO buffers, used when they might be armed, but
816  * not triggered.  Used at init to ensure buffer state, and also user
817  * process close, in case it died while writing to a PIO buffer
818  * Also after errors.
819  */
820 void ipath_disarm_piobufs(struct ipath_devdata *dd, unsigned first,
821                           unsigned cnt)
822 {
823         unsigned i, last = first + cnt;
824         unsigned long flags;
825 
826         ipath_cdbg(PKT, "disarm %u PIObufs first=%u\n", cnt, first);
827         for (i = first; i < last; i++) {
828                 spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags);
829                 /*
830                  * The disarm-related bits are write-only, so it
831                  * is ok to OR them in with our copy of sendctrl
832                  * while we hold the lock.
833                  */
834                 ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
835                         dd->ipath_sendctrl | INFINIPATH_S_DISARM |
836                         (i << INFINIPATH_S_DISARMPIOBUF_SHIFT));
837                 /* can't disarm bufs back-to-back per iba7220 spec */
838                 ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
839                 spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags);
840         }
841         /* on some older chips, update may not happen after cancel */
842         ipath_force_pio_avail_update(dd);
843 }
844 
845 /**
846  * ipath_wait_linkstate - wait for an IB link state change to occur
847  * @dd: the infinipath device
848  * @state: the state to wait for
849  * @msecs: the number of milliseconds to wait
850  *
851  * wait up to msecs milliseconds for IB link state change to occur for
852  * now, take the easy polling route.  Currently used only by
853  * ipath_set_linkstate.  Returns 0 if state reached, otherwise
854  * -ETIMEDOUT state can have multiple states set, for any of several
855  * transitions.
856  */
857 int ipath_wait_linkstate(struct ipath_devdata *dd, u32 state, int msecs)
858 {
859         dd->ipath_state_wanted = state;
860         wait_event_interruptible_timeout(ipath_state_wait,
861                                          (dd->ipath_flags & state),
862                                          msecs_to_jiffies(msecs));
863         dd->ipath_state_wanted = 0;
864 
865         if (!(dd->ipath_flags & state)) {
866                 u64 val;
867                 ipath_cdbg(VERBOSE, "Didn't reach linkstate %s within %u"
868                            " ms\n",
869                            /* test INIT ahead of DOWN, both can be set */
870                            (state & IPATH_LINKINIT) ? "INIT" :
871                            ((state & IPATH_LINKDOWN) ? "DOWN" :
872                             ((state & IPATH_LINKARMED) ? "ARM" : "ACTIVE")),
873                            msecs);
874                 val = ipath_read_kreg64(dd, dd->ipath_kregs->kr_ibcstatus);
875                 ipath_cdbg(VERBOSE, "ibcc=%llx ibcstatus=%llx (%s)\n",
876                            (unsigned long long) ipath_read_kreg64(
877                                    dd, dd->ipath_kregs->kr_ibcctrl),
878                            (unsigned long long) val,
879                            ipath_ibcstatus_str[val & dd->ibcs_lts_mask]);
880         }
881         return (dd->ipath_flags & state) ? 0 : -ETIMEDOUT;
882 }
883 
884 static void decode_sdma_errs(struct ipath_devdata *dd, ipath_err_t err,
885         char *buf, size_t blen)
886 {
887         static const struct {
888                 ipath_err_t err;
889                 const char *msg;
890         } errs[] = {
891                 { INFINIPATH_E_SDMAGENMISMATCH, "SDmaGenMismatch" },
892                 { INFINIPATH_E_SDMAOUTOFBOUND, "SDmaOutOfBound" },
893                 { INFINIPATH_E_SDMATAILOUTOFBOUND, "SDmaTailOutOfBound" },
894                 { INFINIPATH_E_SDMABASE, "SDmaBase" },
895                 { INFINIPATH_E_SDMA1STDESC, "SDma1stDesc" },
896                 { INFINIPATH_E_SDMARPYTAG, "SDmaRpyTag" },
897                 { INFINIPATH_E_SDMADWEN, "SDmaDwEn" },
898                 { INFINIPATH_E_SDMAMISSINGDW, "SDmaMissingDw" },
899                 { INFINIPATH_E_SDMAUNEXPDATA, "SDmaUnexpData" },
900                 { INFINIPATH_E_SDMADESCADDRMISALIGN, "SDmaDescAddrMisalign" },
901                 { INFINIPATH_E_SENDBUFMISUSE, "SendBufMisuse" },
902                 { INFINIPATH_E_SDMADISABLED, "SDmaDisabled" },
903         };
904         int i;
905         int expected;
906         size_t bidx = 0;
907 
908         for (i = 0; i < ARRAY_SIZE(errs); i++) {
909                 expected = (errs[i].err != INFINIPATH_E_SDMADISABLED) ? 0 :
910                         test_bit(IPATH_SDMA_ABORTING, &dd->ipath_sdma_status);
911                 if ((err & errs[i].err) && !expected)
912                         bidx += snprintf(buf + bidx, blen - bidx,
913                                          "%s ", errs[i].msg);
914         }
915 }
916 
917 /*
918  * Decode the error status into strings, deciding whether to always
919  * print * it or not depending on "normal packet errors" vs everything
920  * else.   Return 1 if "real" errors, otherwise 0 if only packet
921  * errors, so caller can decide what to print with the string.
922  */
923 int ipath_decode_err(struct ipath_devdata *dd, char *buf, size_t blen,
924         ipath_err_t err)
925 {
926         int iserr = 1;
927         *buf = '\0';
928         if (err & INFINIPATH_E_PKTERRS) {
929                 if (!(err & ~INFINIPATH_E_PKTERRS))
930                         iserr = 0; // if only packet errors.
931                 if (ipath_debug & __IPATH_ERRPKTDBG) {
932                         if (err & INFINIPATH_E_REBP)
933                                 strlcat(buf, "EBP ", blen);
934                         if (err & INFINIPATH_E_RVCRC)
935                                 strlcat(buf, "VCRC ", blen);
936                         if (err & INFINIPATH_E_RICRC) {
937                                 strlcat(buf, "CRC ", blen);
938                                 // clear for check below, so only once
939                                 err &= INFINIPATH_E_RICRC;
940                         }
941                         if (err & INFINIPATH_E_RSHORTPKTLEN)
942                                 strlcat(buf, "rshortpktlen ", blen);
943                         if (err & INFINIPATH_E_SDROPPEDDATAPKT)
944                                 strlcat(buf, "sdroppeddatapkt ", blen);
945                         if (err & INFINIPATH_E_SPKTLEN)
946                                 strlcat(buf, "spktlen ", blen);
947                 }
948                 if ((err & INFINIPATH_E_RICRC) &&
949                         !(err&(INFINIPATH_E_RVCRC|INFINIPATH_E_REBP)))
950                         strlcat(buf, "CRC ", blen);
951                 if (!iserr)
952                         goto done;
953         }
954         if (err & INFINIPATH_E_RHDRLEN)
955                 strlcat(buf, "rhdrlen ", blen);
956         if (err & INFINIPATH_E_RBADTID)
957                 strlcat(buf, "rbadtid ", blen);
958         if (err & INFINIPATH_E_RBADVERSION)
959                 strlcat(buf, "rbadversion ", blen);
960         if (err & INFINIPATH_E_RHDR)
961                 strlcat(buf, "rhdr ", blen);
962         if (err & INFINIPATH_E_SENDSPECIALTRIGGER)
963                 strlcat(buf, "sendspecialtrigger ", blen);
964         if (err & INFINIPATH_E_RLONGPKTLEN)
965                 strlcat(buf, "rlongpktlen ", blen);
966         if (err & INFINIPATH_E_RMAXPKTLEN)
967                 strlcat(buf, "rmaxpktlen ", blen);
968         if (err & INFINIPATH_E_RMINPKTLEN)
969                 strlcat(buf, "rminpktlen ", blen);
970         if (err & INFINIPATH_E_SMINPKTLEN)
971                 strlcat(buf, "sminpktlen ", blen);
972         if (err & INFINIPATH_E_RFORMATERR)
973                 strlcat(buf, "rformaterr ", blen);
974         if (err & INFINIPATH_E_RUNSUPVL)
975                 strlcat(buf, "runsupvl ", blen);
976         if (err & INFINIPATH_E_RUNEXPCHAR)
977                 strlcat(buf, "runexpchar ", blen);
978         if (err & INFINIPATH_E_RIBFLOW)
979                 strlcat(buf, "ribflow ", blen);
980         if (err & INFINIPATH_E_SUNDERRUN)
981                 strlcat(buf, "sunderrun ", blen);
982         if (err & INFINIPATH_E_SPIOARMLAUNCH)
983                 strlcat(buf, "spioarmlaunch ", blen);
984         if (err & INFINIPATH_E_SUNEXPERRPKTNUM)
985                 strlcat(buf, "sunexperrpktnum ", blen);
986         if (err & INFINIPATH_E_SDROPPEDSMPPKT)
987                 strlcat(buf, "sdroppedsmppkt ", blen);
988         if (err & INFINIPATH_E_SMAXPKTLEN)
989                 strlcat(buf, "smaxpktlen ", blen);
990         if (err & INFINIPATH_E_SUNSUPVL)
991                 strlcat(buf, "sunsupVL ", blen);
992         if (err & INFINIPATH_E_INVALIDADDR)
993                 strlcat(buf, "invalidaddr ", blen);
994         if (err & INFINIPATH_E_RRCVEGRFULL)
995                 strlcat(buf, "rcvegrfull ", blen);
996         if (err & INFINIPATH_E_RRCVHDRFULL)
997                 strlcat(buf, "rcvhdrfull ", blen);
998         if (err & INFINIPATH_E_IBSTATUSCHANGED)
999                 strlcat(buf, "ibcstatuschg ", blen);
1000         if (err & INFINIPATH_E_RIBLOSTLINK)
1001                 strlcat(buf, "riblostlink ", blen);
1002         if (err & INFINIPATH_E_HARDWARE)
1003                 strlcat(buf, "hardware ", blen);
1004         if (err & INFINIPATH_E_RESET)
1005                 strlcat(buf, "reset ", blen);
1006         if (err & INFINIPATH_E_SDMAERRS)
1007                 decode_sdma_errs(dd, err, buf, blen);
1008         if (err & INFINIPATH_E_INVALIDEEPCMD)
1009                 strlcat(buf, "invalideepromcmd ", blen);
1010 done:
1011         return iserr;
1012 }
1013 
1014 /**
1015  * get_rhf_errstring - decode RHF errors
1016  * @err: the err number
1017  * @msg: the output buffer
1018  * @len: the length of the output buffer
1019  *
1020  * only used one place now, may want more later
1021  */
1022 static void get_rhf_errstring(u32 err, char *msg, size_t len)
1023 {
1024         /* if no errors, and so don't need to check what's first */
1025         *msg = '\0';
1026 
1027         if (err & INFINIPATH_RHF_H_ICRCERR)
1028                 strlcat(msg, "icrcerr ", len);
1029         if (err & INFINIPATH_RHF_H_VCRCERR)
1030                 strlcat(msg, "vcrcerr ", len);
1031         if (err & INFINIPATH_RHF_H_PARITYERR)
1032                 strlcat(msg, "parityerr ", len);
1033         if (err & INFINIPATH_RHF_H_LENERR)
1034                 strlcat(msg, "lenerr ", len);
1035         if (err & INFINIPATH_RHF_H_MTUERR)
1036                 strlcat(msg, "mtuerr ", len);
1037         if (err & INFINIPATH_RHF_H_IHDRERR)
1038                 /* infinipath hdr checksum error */
1039                 strlcat(msg, "ipathhdrerr ", len);
1040         if (err & INFINIPATH_RHF_H_TIDERR)
1041                 strlcat(msg, "tiderr ", len);
1042         if (err & INFINIPATH_RHF_H_MKERR)
1043                 /* bad port, offset, etc. */
1044                 strlcat(msg, "invalid ipathhdr ", len);
1045         if (err & INFINIPATH_RHF_H_IBERR)
1046                 strlcat(msg, "iberr ", len);
1047         if (err & INFINIPATH_RHF_L_SWA)
1048                 strlcat(msg, "swA ", len);
1049         if (err & INFINIPATH_RHF_L_SWB)
1050                 strlcat(msg, "swB ", len);
1051 }
1052 
1053 /**
1054  * ipath_get_egrbuf - get an eager buffer
1055  * @dd: the infinipath device
1056  * @bufnum: the eager buffer to get
1057  *
1058  * must only be called if ipath_pd[port] is known to be allocated
1059  */
1060 static inline void *ipath_get_egrbuf(struct ipath_devdata *dd, u32 bufnum)
1061 {
1062         return dd->ipath_port0_skbinfo ?
1063                 (void *) dd->ipath_port0_skbinfo[bufnum].skb->data : NULL;
1064 }
1065 
1066 /**
1067  * ipath_alloc_skb - allocate an skb and buffer with possible constraints
1068  * @dd: the infinipath device
1069  * @gfp_mask: the sk_buff SFP mask
1070  */
1071 struct sk_buff *ipath_alloc_skb(struct ipath_devdata *dd,
1072                                 gfp_t gfp_mask)
1073 {
1074         struct sk_buff *skb;
1075         u32 len;
1076 
1077         /*
1078          * Only fully supported way to handle this is to allocate lots
1079          * extra, align as needed, and then do skb_reserve().  That wastes
1080          * a lot of memory...  I'll have to hack this into infinipath_copy
1081          * also.
1082          */
1083 
1084         /*
1085          * We need 2 extra bytes for ipath_ether data sent in the
1086          * key header.  In order to keep everything dword aligned,
1087          * we'll reserve 4 bytes.
1088          */
1089         len = dd->ipath_ibmaxlen + 4;
1090 
1091         if (dd->ipath_flags & IPATH_4BYTE_TID) {
1092                 /* We need a 2KB multiple alignment, and there is no way
1093                  * to do it except to allocate extra and then skb_reserve
1094                  * enough to bring it up to the right alignment.
1095                  */
1096                 len += 2047;
1097         }
1098 
1099         skb = __dev_alloc_skb(len, gfp_mask);
1100         if (!skb) {
1101                 ipath_dev_err(dd, "Failed to allocate skbuff, length %u\n",
1102                               len);
1103                 goto bail;
1104         }
1105 
1106         skb_reserve(skb, 4);
1107 
1108         if (dd->ipath_flags & IPATH_4BYTE_TID) {
1109                 u32 una = (unsigned long)skb->data & 2047;
1110                 if (una)
1111                         skb_reserve(skb, 2048 - una);
1112         }
1113 
1114 bail:
1115         return skb;
1116 }
1117 
1118 static void ipath_rcv_hdrerr(struct ipath_devdata *dd,
1119                              u32 eflags,
1120                              u32 l,
1121                              u32 etail,
1122                              __le32 *rhf_addr,
1123                              struct ipath_message_header *hdr)
1124 {
1125         char emsg[128];
1126 
1127         get_rhf_errstring(eflags, emsg, sizeof emsg);
1128         ipath_cdbg(PKT, "RHFerrs %x hdrqtail=%x typ=%u "
1129                    "tlen=%x opcode=%x egridx=%x: %s\n",
1130                    eflags, l,
1131                    ipath_hdrget_rcv_type(rhf_addr),
1132                    ipath_hdrget_length_in_bytes(rhf_addr),
1133                    be32_to_cpu(hdr->bth[0]) >> 24,
1134                    etail, emsg);
1135 
1136         /* Count local link integrity errors. */
1137         if (eflags & (INFINIPATH_RHF_H_ICRCERR | INFINIPATH_RHF_H_VCRCERR)) {
1138                 u8 n = (dd->ipath_ibcctrl >>
1139                         INFINIPATH_IBCC_PHYERRTHRESHOLD_SHIFT) &
1140                         INFINIPATH_IBCC_PHYERRTHRESHOLD_MASK;
1141 
1142                 if (++dd->ipath_lli_counter > n) {
1143                         dd->ipath_lli_counter = 0;
1144                         dd->ipath_lli_errors++;
1145                 }
1146         }
1147 }
1148 
1149 /*
1150  * ipath_kreceive - receive a packet
1151  * @pd: the infinipath port
1152  *
1153  * called from interrupt handler for errors or receive interrupt
1154  */
1155 void ipath_kreceive(struct ipath_portdata *pd)
1156 {
1157         struct ipath_devdata *dd = pd->port_dd;
1158         __le32 *rhf_addr;
1159         void *ebuf;
1160         const u32 rsize = dd->ipath_rcvhdrentsize;      /* words */
1161         const u32 maxcnt = dd->ipath_rcvhdrcnt * rsize; /* words */
1162         u32 etail = -1, l, hdrqtail;
1163         struct ipath_message_header *hdr;
1164         u32 eflags, i, etype, tlen, pkttot = 0, updegr = 0, reloop = 0;
1165         static u64 totcalls;    /* stats, may eventually remove */
1166         int last;
1167 
1168         l = pd->port_head;
1169         rhf_addr = (__le32 *) pd->port_rcvhdrq + l + dd->ipath_rhf_offset;
1170         if (dd->ipath_flags & IPATH_NODMA_RTAIL) {
1171                 u32 seq = ipath_hdrget_seq(rhf_addr);
1172 
1173                 if (seq != pd->port_seq_cnt)
1174                         goto bail;
1175                 hdrqtail = 0;
1176         } else {
1177                 hdrqtail = ipath_get_rcvhdrtail(pd);
1178                 if (l == hdrqtail)
1179                         goto bail;
1180                 smp_rmb();
1181         }
1182 
1183 reloop:
1184         for (last = 0, i = 1; !last; i += !last) {
1185                 hdr = dd->ipath_f_get_msgheader(dd, rhf_addr);
1186                 eflags = ipath_hdrget_err_flags(rhf_addr);
1187                 etype = ipath_hdrget_rcv_type(rhf_addr);
1188                 /* total length */
1189                 tlen = ipath_hdrget_length_in_bytes(rhf_addr);
1190                 ebuf = NULL;
1191                 if ((dd->ipath_flags & IPATH_NODMA_RTAIL) ?
1192                     ipath_hdrget_use_egr_buf(rhf_addr) :
1193                     (etype != RCVHQ_RCV_TYPE_EXPECTED)) {
1194                         /*
1195                          * It turns out that the chip uses an eager buffer
1196                          * for all non-expected packets, whether it "needs"
1197                          * one or not.  So always get the index, but don't
1198                          * set ebuf (so we try to copy data) unless the
1199                          * length requires it.
1200                          */
1201                         etail = ipath_hdrget_index(rhf_addr);
1202                         updegr = 1;
1203                         if (tlen > sizeof(*hdr) ||
1204                             etype == RCVHQ_RCV_TYPE_NON_KD)
1205                                 ebuf = ipath_get_egrbuf(dd, etail);
1206                 }
1207 
1208                 /*
1209                  * both tiderr and ipathhdrerr are set for all plain IB
1210                  * packets; only ipathhdrerr should be set.
1211                  */
1212 
1213                 if (etype != RCVHQ_RCV_TYPE_NON_KD &&
1214                     etype != RCVHQ_RCV_TYPE_ERROR &&
1215                     ipath_hdrget_ipath_ver(hdr->iph.ver_port_tid_offset) !=
1216                     IPS_PROTO_VERSION)
1217                         ipath_cdbg(PKT, "Bad InfiniPath protocol version "
1218                                    "%x\n", etype);
1219 
1220                 if (unlikely(eflags))
1221                         ipath_rcv_hdrerr(dd, eflags, l, etail, rhf_addr, hdr);
1222                 else if (etype == RCVHQ_RCV_TYPE_NON_KD) {
1223                         ipath_ib_rcv(dd->verbs_dev, (u32 *)hdr, ebuf, tlen);
1224                         if (dd->ipath_lli_counter)
1225                                 dd->ipath_lli_counter--;
1226                 } else if (etype == RCVHQ_RCV_TYPE_EAGER) {
1227                         u8 opcode = be32_to_cpu(hdr->bth[0]) >> 24;
1228                         u32 qp = be32_to_cpu(hdr->bth[1]) & 0xffffff;
1229                         ipath_cdbg(PKT, "typ %x, opcode %x (eager, "
1230                                    "qp=%x), len %x; ignored\n",
1231                                    etype, opcode, qp, tlen);
1232                 }
1233                 else if (etype == RCVHQ_RCV_TYPE_EXPECTED)
1234                         ipath_dbg("Bug: Expected TID, opcode %x; ignored\n",
1235                                   be32_to_cpu(hdr->bth[0]) >> 24);
1236                 else {
1237                         /*
1238                          * error packet, type of error unknown.
1239                          * Probably type 3, but we don't know, so don't
1240                          * even try to print the opcode, etc.
1241                          * Usually caused by a "bad packet", that has no
1242                          * BTH, when the LRH says it should.
1243                          */
1244                         ipath_cdbg(ERRPKT, "Error Pkt, but no eflags! egrbuf"
1245                                   " %x, len %x hdrq+%x rhf: %Lx\n",
1246                                   etail, tlen, l, (unsigned long long)
1247                                   le64_to_cpu(*(__le64 *) rhf_addr));
1248                         if (ipath_debug & __IPATH_ERRPKTDBG) {
1249                                 u32 j, *d, dw = rsize-2;
1250                                 if (rsize > (tlen>>2))
1251                                         dw = tlen>>2;
1252                                 d = (u32 *)hdr;
1253                                 printk(KERN_DEBUG "EPkt rcvhdr(%x dw):\n",
1254                                         dw);
1255                                 for (j = 0; j < dw; j++)
1256                                         printk(KERN_DEBUG "%8x%s", d[j],
1257                                                 (j%8) == 7 ? "\n" : " ");
1258                                 printk(KERN_DEBUG ".\n");
1259                         }
1260                 }
1261                 l += rsize;
1262                 if (l >= maxcnt)
1263                         l = 0;
1264                 rhf_addr = (__le32 *) pd->port_rcvhdrq +
1265                         l + dd->ipath_rhf_offset;
1266                 if (dd->ipath_flags & IPATH_NODMA_RTAIL) {
1267                         u32 seq = ipath_hdrget_seq(rhf_addr);
1268 
1269                         if (++pd->port_seq_cnt > 13)
1270                                 pd->port_seq_cnt = 1;
1271                         if (seq != pd->port_seq_cnt)
1272                                 last = 1;
1273                 } else if (l == hdrqtail)
1274                         last = 1;
1275                 /*
1276                  * update head regs on last packet, and every 16 packets.
1277                  * Reduce bus traffic, while still trying to prevent
1278                  * rcvhdrq overflows, for when the queue is nearly full
1279                  */
1280                 if (last || !(i & 0xf)) {
1281                         u64 lval = l;
1282 
1283                         /* request IBA6120 and 7220 interrupt only on last */
1284                         if (last)
1285                                 lval |= dd->ipath_rhdrhead_intr_off;
1286                         ipath_write_ureg(dd, ur_rcvhdrhead, lval,
1287                                 pd->port_port);
1288                         if (updegr) {
1289                                 ipath_write_ureg(dd, ur_rcvegrindexhead,
1290                                                  etail, pd->port_port);
1291                                 updegr = 0;
1292                         }
1293                 }
1294         }
1295 
1296         if (!dd->ipath_rhdrhead_intr_off && !reloop &&
1297             !(dd->ipath_flags & IPATH_NODMA_RTAIL)) {
1298                 /* IBA6110 workaround; we can have a race clearing chip
1299                  * interrupt with another interrupt about to be delivered,
1300                  * and can clear it before it is delivered on the GPIO
1301                  * workaround.  By doing the extra check here for the
1302                  * in-memory tail register updating while we were doing
1303                  * earlier packets, we "almost" guarantee we have covered
1304                  * that case.
1305                  */
1306                 u32 hqtail = ipath_get_rcvhdrtail(pd);
1307                 if (hqtail != hdrqtail) {
1308                         hdrqtail = hqtail;
1309                         reloop = 1; /* loop 1 extra time at most */
1310                         goto reloop;
1311                 }
1312         }
1313 
1314         pkttot += i;
1315 
1316         pd->port_head = l;
1317 
1318         if (pkttot > ipath_stats.sps_maxpkts_call)
1319                 ipath_stats.sps_maxpkts_call = pkttot;
1320         ipath_stats.sps_port0pkts += pkttot;
1321         ipath_stats.sps_avgpkts_call =
1322                 ipath_stats.sps_port0pkts / ++totcalls;
1323 
1324 bail:;
1325 }
1326 
1327 /**
1328  * ipath_update_pio_bufs - update shadow copy of the PIO availability map
1329  * @dd: the infinipath device
1330  *
1331  * called whenever our local copy indicates we have run out of send buffers
1332  * NOTE: This can be called from interrupt context by some code
1333  * and from non-interrupt context by ipath_getpiobuf().
1334  */
1335 
1336 static void ipath_update_pio_bufs(struct ipath_devdata *dd)
1337 {
1338         unsigned long flags;
1339         int i;
1340         const unsigned piobregs = (unsigned)dd->ipath_pioavregs;
1341 
1342         /* If the generation (check) bits have changed, then we update the
1343          * busy bit for the corresponding PIO buffer.  This algorithm will
1344          * modify positions to the value they already have in some cases
1345          * (i.e., no change), but it's faster than changing only the bits
1346          * that have changed.
1347          *
1348          * We would like to do this atomicly, to avoid spinlocks in the
1349          * critical send path, but that's not really possible, given the
1350          * type of changes, and that this routine could be called on
1351          * multiple cpu's simultaneously, so we lock in this routine only,
1352          * to avoid conflicting updates; all we change is the shadow, and
1353          * it's a single 64 bit memory location, so by definition the update
1354          * is atomic in terms of what other cpu's can see in testing the
1355          * bits.  The spin_lock overhead isn't too bad, since it only
1356          * happens when all buffers are in use, so only cpu overhead, not
1357          * latency or bandwidth is affected.
1358          */
1359         if (!dd->ipath_pioavailregs_dma) {
1360                 ipath_dbg("Update shadow pioavail, but regs_dma NULL!\n");
1361                 return;
1362         }
1363         if (ipath_debug & __IPATH_VERBDBG) {
1364                 /* only if packet debug and verbose */
1365                 volatile __le64 *dma = dd->ipath_pioavailregs_dma;
1366                 unsigned long *shadow = dd->ipath_pioavailshadow;
1367 
1368                 ipath_cdbg(PKT, "Refill avail, dma0=%llx shad0=%lx, "
1369                            "d1=%llx s1=%lx, d2=%llx s2=%lx, d3=%llx "
1370                            "s3=%lx\n",
1371                            (unsigned long long) le64_to_cpu(dma[0]),
1372                            shadow[0],
1373                            (unsigned long long) le64_to_cpu(dma[1]),
1374                            shadow[1],
1375                            (unsigned long long) le64_to_cpu(dma[2]),
1376                            shadow[2],
1377                            (unsigned long long) le64_to_cpu(dma[3]),
1378                            shadow[3]);
1379                 if (piobregs > 4)
1380                         ipath_cdbg(
1381                                 PKT, "2nd group, dma4=%llx shad4=%lx, "
1382                                 "d5=%llx s5=%lx, d6=%llx s6=%lx, "
1383                                 "d7=%llx s7=%lx\n",
1384                                 (unsigned long long) le64_to_cpu(dma[4]),
1385                                 shadow[4],
1386                                 (unsigned long long) le64_to_cpu(dma[5]),
1387                                 shadow[5],
1388                                 (unsigned long long) le64_to_cpu(dma[6]),
1389                                 shadow[6],
1390                                 (unsigned long long) le64_to_cpu(dma[7]),
1391                                 shadow[7]);
1392         }
1393         spin_lock_irqsave(&ipath_pioavail_lock, flags);
1394         for (i = 0; i < piobregs; i++) {
1395                 u64 pchbusy, pchg, piov, pnew;
1396                 /*
1397                  * Chip Errata: bug 6641; even and odd qwords>3 are swapped
1398                  */
1399                 if (i > 3 && (dd->ipath_flags & IPATH_SWAP_PIOBUFS))
1400                         piov = le64_to_cpu(dd->ipath_pioavailregs_dma[i ^ 1]);
1401                 else
1402                         piov = le64_to_cpu(dd->ipath_pioavailregs_dma[i]);
1403                 pchg = dd->ipath_pioavailkernel[i] &
1404                         ~(dd->ipath_pioavailshadow[i] ^ piov);
1405                 pchbusy = pchg << INFINIPATH_SENDPIOAVAIL_BUSY_SHIFT;
1406                 if (pchg && (pchbusy & dd->ipath_pioavailshadow[i])) {
1407                         pnew = dd->ipath_pioavailshadow[i] & ~pchbusy;
1408                         pnew |= piov & pchbusy;
1409                         dd->ipath_pioavailshadow[i] = pnew;
1410                 }
1411         }
1412         spin_unlock_irqrestore(&ipath_pioavail_lock, flags);
1413 }
1414 
1415 /*
1416  * used to force update of pioavailshadow if we can't get a pio buffer.
1417  * Needed primarily due to exitting freeze mode after recovering
1418  * from errors.  Done lazily, because it's safer (known to not
1419  * be writing pio buffers).
1420  */
1421 static void ipath_reset_availshadow(struct ipath_devdata *dd)
1422 {
1423         int i, im;
1424         unsigned long flags;
1425 
1426         spin_lock_irqsave(&ipath_pioavail_lock, flags);
1427         for (i = 0; i < dd->ipath_pioavregs; i++) {
1428                 u64 val, oldval;
1429                 /* deal with 6110 chip bug on high register #s */
1430                 im = (i > 3 && (dd->ipath_flags & IPATH_SWAP_PIOBUFS)) ?
1431                         i ^ 1 : i;
1432                 val = le64_to_cpu(dd->ipath_pioavailregs_dma[im]);
1433                 /*
1434                  * busy out the buffers not in the kernel avail list,
1435                  * without changing the generation bits.
1436                  */
1437                 oldval = dd->ipath_pioavailshadow[i];
1438                 dd->ipath_pioavailshadow[i] = val |
1439                         ((~dd->ipath_pioavailkernel[i] <<
1440                         INFINIPATH_SENDPIOAVAIL_BUSY_SHIFT) &
1441                         0xaaaaaaaaaaaaaaaaULL); /* All BUSY bits in qword */
1442                 if (oldval != dd->ipath_pioavailshadow[i])
1443                         ipath_dbg("shadow[%d] was %Lx, now %lx\n",
1444                                 i, (unsigned long long) oldval,
1445                                 dd->ipath_pioavailshadow[i]);
1446         }
1447         spin_unlock_irqrestore(&ipath_pioavail_lock, flags);
1448 }
1449 
1450 /**
1451  * ipath_setrcvhdrsize - set the receive header size
1452  * @dd: the infinipath device
1453  * @rhdrsize: the receive header size
1454  *
1455  * called from user init code, and also layered driver init
1456  */
1457 int ipath_setrcvhdrsize(struct ipath_devdata *dd, unsigned rhdrsize)
1458 {
1459         int ret = 0;
1460 
1461         if (dd->ipath_flags & IPATH_RCVHDRSZ_SET) {
1462                 if (dd->ipath_rcvhdrsize != rhdrsize) {
1463                         dev_info(&dd->pcidev->dev,
1464                                  "Error: can't set protocol header "
1465                                  "size %u, already %u\n",
1466                                  rhdrsize, dd->ipath_rcvhdrsize);
1467                         ret = -EAGAIN;
1468                 } else
1469                         ipath_cdbg(VERBOSE, "Reuse same protocol header "
1470                                    "size %u\n", dd->ipath_rcvhdrsize);
1471         } else if (rhdrsize > (dd->ipath_rcvhdrentsize -
1472                                (sizeof(u64) / sizeof(u32)))) {
1473                 ipath_dbg("Error: can't set protocol header size %u "
1474                           "(> max %u)\n", rhdrsize,
1475                           dd->ipath_rcvhdrentsize -
1476                           (u32) (sizeof(u64) / sizeof(u32)));
1477                 ret = -EOVERFLOW;
1478         } else {
1479                 dd->ipath_flags |= IPATH_RCVHDRSZ_SET;
1480                 dd->ipath_rcvhdrsize = rhdrsize;
1481                 ipath_write_kreg(dd, dd->ipath_kregs->kr_rcvhdrsize,
1482                                  dd->ipath_rcvhdrsize);
1483                 ipath_cdbg(VERBOSE, "Set protocol header size to %u\n",
1484                            dd->ipath_rcvhdrsize);
1485         }
1486         return ret;
1487 }
1488 
1489 /*
1490  * debugging code and stats updates if no pio buffers available.
1491  */
1492 static noinline void no_pio_bufs(struct ipath_devdata *dd)
1493 {
1494         unsigned long *shadow = dd->ipath_pioavailshadow;
1495         __le64 *dma = (__le64 *)dd->ipath_pioavailregs_dma;
1496 
1497         dd->ipath_upd_pio_shadow = 1;
1498 
1499         /*
1500          * not atomic, but if we lose a stat count in a while, that's OK
1501          */
1502         ipath_stats.sps_nopiobufs++;
1503         if (!(++dd->ipath_consec_nopiobuf % 100000)) {
1504                 ipath_force_pio_avail_update(dd); /* at start */
1505                 ipath_dbg("%u tries no piobufavail ts%lx; dmacopy: "
1506                         "%llx %llx %llx %llx\n"
1507                         "ipath  shadow:  %lx %lx %lx %lx\n",
1508                         dd->ipath_consec_nopiobuf,
1509                         (unsigned long)get_cycles(),
1510                         (unsigned long long) le64_to_cpu(dma[0]),
1511                         (unsigned long long) le64_to_cpu(dma[1]),
1512                         (unsigned long long) le64_to_cpu(dma[2]),
1513                         (unsigned long long) le64_to_cpu(dma[3]),
1514                         shadow[0], shadow[1], shadow[2], shadow[3]);
1515                 /*
1516                  * 4 buffers per byte, 4 registers above, cover rest
1517                  * below
1518                  */
1519                 if ((dd->ipath_piobcnt2k + dd->ipath_piobcnt4k) >
1520                     (sizeof(shadow[0]) * 4 * 4))
1521                         ipath_dbg("2nd group: dmacopy: "
1522                                   "%llx %llx %llx %llx\n"
1523                                   "ipath  shadow:  %lx %lx %lx %lx\n",
1524                                   (unsigned long long)le64_to_cpu(dma[4]),
1525                                   (unsigned long long)le64_to_cpu(dma[5]),
1526                                   (unsigned long long)le64_to_cpu(dma[6]),
1527                                   (unsigned long long)le64_to_cpu(dma[7]),
1528                                   shadow[4], shadow[5], shadow[6], shadow[7]);
1529 
1530                 /* at end, so update likely happened */
1531                 ipath_reset_availshadow(dd);
1532         }
1533 }
1534 
1535 /*
1536  * common code for normal driver pio buffer allocation, and reserved
1537  * allocation.
1538  *
1539  * do appropriate marking as busy, etc.
1540  * returns buffer number if one found (>=0), negative number is error.
1541  */
1542 static u32 __iomem *ipath_getpiobuf_range(struct ipath_devdata *dd,
1543         u32 *pbufnum, u32 first, u32 last, u32 firsti)
1544 {
1545         int i, j, updated = 0;
1546         unsigned piobcnt;
1547         unsigned long flags;
1548         unsigned long *shadow = dd->ipath_pioavailshadow;
1549         u32 __iomem *buf;
1550 
1551         piobcnt = last - first;
1552         if (dd->ipath_upd_pio_shadow) {
1553                 /*
1554                  * Minor optimization.  If we had no buffers on last call,
1555                  * start out by doing the update; continue and do scan even
1556                  * if no buffers were updated, to be paranoid
1557                  */
1558                 ipath_update_pio_bufs(dd);
1559                 updated++;
1560                 i = first;
1561         } else
1562                 i = firsti;
1563 rescan:
1564         /*
1565          * while test_and_set_bit() is atomic, we do that and then the
1566          * change_bit(), and the pair is not.  See if this is the cause
1567          * of the remaining armlaunch errors.
1568          */
1569         spin_lock_irqsave(&ipath_pioavail_lock, flags);
1570         for (j = 0; j < piobcnt; j++, i++) {
1571                 if (i >= last)
1572                         i = first;
1573                 if (__test_and_set_bit((2 * i) + 1, shadow))
1574                         continue;
1575                 /* flip generation bit */
1576                 __change_bit(2 * i, shadow);
1577                 break;
1578         }
1579         spin_unlock_irqrestore(&ipath_pioavail_lock, flags);
1580 
1581         if (j == piobcnt) {
1582                 if (!updated) {
1583                         /*
1584                          * first time through; shadow exhausted, but may be
1585                          * buffers available, try an update and then rescan.
1586                          */
1587                         ipath_update_pio_bufs(dd);
1588                         updated++;
1589                         i = first;
1590                         goto rescan;
1591                 } else if (updated == 1 && piobcnt <=
1592                         ((dd->ipath_sendctrl
1593                         >> INFINIPATH_S_UPDTHRESH_SHIFT) &
1594                         INFINIPATH_S_UPDTHRESH_MASK)) {
1595                         /*
1596                          * for chips supporting and using the update
1597                          * threshold we need to force an update of the
1598                          * in-memory copy if the count is less than the
1599                          * thershold, then check one more time.
1600                          */
1601                         ipath_force_pio_avail_update(dd);
1602                         ipath_update_pio_bufs(dd);
1603                         updated++;
1604                         i = first;
1605                         goto rescan;
1606                 }
1607 
1608                 no_pio_bufs(dd);
1609                 buf = NULL;
1610         } else {
1611                 if (i < dd->ipath_piobcnt2k)
1612                         buf = (u32 __iomem *) (dd->ipath_pio2kbase +
1613                                                i * dd->ipath_palign);
1614                 else
1615                         buf = (u32 __iomem *)
1616                                 (dd->ipath_pio4kbase +
1617                                  (i - dd->ipath_piobcnt2k) * dd->ipath_4kalign);
1618                 if (pbufnum)
1619                         *pbufnum = i;
1620         }
1621 
1622         return buf;
1623 }
1624 
1625 /**
1626  * ipath_getpiobuf - find an available pio buffer
1627  * @dd: the infinipath device
1628  * @plen: the size of the PIO buffer needed in 32-bit words
1629  * @pbufnum: the buffer number is placed here
1630  */
1631 u32 __iomem *ipath_getpiobuf(struct ipath_devdata *dd, u32 plen, u32 *pbufnum)
1632 {
1633         u32 __iomem *buf;
1634         u32 pnum, nbufs;
1635         u32 first, lasti;
1636 
1637         if (plen + 1 >= IPATH_SMALLBUF_DWORDS) {
1638                 first = dd->ipath_piobcnt2k;
1639                 lasti = dd->ipath_lastpioindexl;
1640         } else {
1641                 first = 0;
1642                 lasti = dd->ipath_lastpioindex;
1643         }
1644         nbufs = dd->ipath_piobcnt2k + dd->ipath_piobcnt4k;
1645         buf = ipath_getpiobuf_range(dd, &pnum, first, nbufs, lasti);
1646 
1647         if (buf) {
1648                 /*
1649                  * Set next starting place.  It's just an optimization,
1650                  * it doesn't matter who wins on this, so no locking
1651                  */
1652                 if (plen + 1 >= IPATH_SMALLBUF_DWORDS)
1653                         dd->ipath_lastpioindexl = pnum + 1;
1654                 else
1655                         dd->ipath_lastpioindex = pnum + 1;
1656                 if (dd->ipath_upd_pio_shadow)
1657                         dd->ipath_upd_pio_shadow = 0;
1658                 if (dd->ipath_consec_nopiobuf)
1659                         dd->ipath_consec_nopiobuf = 0;
1660                 ipath_cdbg(VERBOSE, "Return piobuf%u %uk @ %p\n",
1661                            pnum, (pnum < dd->ipath_piobcnt2k) ? 2 : 4, buf);
1662                 if (pbufnum)
1663                         *pbufnum = pnum;
1664 
1665         }
1666         return buf;
1667 }
1668 
1669 /**
1670  * ipath_chg_pioavailkernel - change which send buffers are available for kernel
1671  * @dd: the infinipath device
1672  * @start: the starting send buffer number
1673  * @len: the number of send buffers
1674  * @avail: true if the buffers are available for kernel use, false otherwise
1675  */
1676 void ipath_chg_pioavailkernel(struct ipath_devdata *dd, unsigned start,
1677                               unsigned len, int avail)
1678 {
1679         unsigned long flags;
1680         unsigned end, cnt = 0;
1681 
1682         /* There are two bits per send buffer (busy and generation) */
1683         start *= 2;
1684         end = start + len * 2;
1685 
1686         spin_lock_irqsave(&ipath_pioavail_lock, flags);
1687         /* Set or clear the busy bit in the shadow. */
1688         while (start < end) {
1689                 if (avail) {
1690                         unsigned long dma;
1691                         int i, im;
1692                         /*
1693                          * the BUSY bit will never be set, because we disarm
1694                          * the user buffers before we hand them back to the
1695                          * kernel.  We do have to make sure the generation
1696                          * bit is set correctly in shadow, since it could
1697                          * have changed many times while allocated to user.
1698                          * We can't use the bitmap functions on the full
1699                          * dma array because it is always little-endian, so
1700                          * we have to flip to host-order first.
1701                          * BITS_PER_LONG is slightly wrong, since it's
1702                          * always 64 bits per register in chip...
1703                          * We only work on 64 bit kernels, so that's OK.
1704                          */
1705                         /* deal with 6110 chip bug on high register #s */
1706                         i = start / BITS_PER_LONG;
1707                         im = (i > 3 && (dd->ipath_flags & IPATH_SWAP_PIOBUFS)) ?
1708                                 i ^ 1 : i;
1709                         __clear_bit(INFINIPATH_SENDPIOAVAIL_BUSY_SHIFT
1710                                 + start, dd->ipath_pioavailshadow);
1711                         dma = (unsigned long) le64_to_cpu(
1712                                 dd->ipath_pioavailregs_dma[im]);
1713                         if (test_bit((INFINIPATH_SENDPIOAVAIL_CHECK_SHIFT
1714                                 + start) % BITS_PER_LONG, &dma))
1715                                 __set_bit(INFINIPATH_SENDPIOAVAIL_CHECK_SHIFT
1716                                         + start, dd->ipath_pioavailshadow);
1717                         else
1718                                 __clear_bit(INFINIPATH_SENDPIOAVAIL_CHECK_SHIFT
1719                                         + start, dd->ipath_pioavailshadow);
1720                         __set_bit(start, dd->ipath_pioavailkernel);
1721                 } else {
1722                         __set_bit(start + INFINIPATH_SENDPIOAVAIL_BUSY_SHIFT,
1723                                 dd->ipath_pioavailshadow);
1724                         __clear_bit(start, dd->ipath_pioavailkernel);
1725                 }
1726                 start += 2;
1727         }
1728 
1729         if (dd->ipath_pioupd_thresh) {
1730                 end = 2 * (dd->ipath_piobcnt2k + dd->ipath_piobcnt4k);
1731                 cnt = bitmap_weight(dd->ipath_pioavailkernel, end);
1732         }
1733         spin_unlock_irqrestore(&ipath_pioavail_lock, flags);
1734 
1735         /*
1736          * When moving buffers from kernel to user, if number assigned to
1737          * the user is less than the pio update threshold, and threshold
1738          * is supported (cnt was computed > 0), drop the update threshold
1739          * so we update at least once per allocated number of buffers.
1740          * In any case, if the kernel buffers are less than the threshold,
1741          * drop the threshold.  We don't bother increasing it, having once
1742          * decreased it, since it would typically just cycle back and forth.
1743          * If we don't decrease below buffers in use, we can wait a long
1744          * time for an update, until some other context uses PIO buffers.
1745          */
1746         if (!avail && len < cnt)
1747                 cnt = len;
1748         if (cnt < dd->ipath_pioupd_thresh) {
1749                 dd->ipath_pioupd_thresh = cnt;
1750                 ipath_dbg("Decreased pio update threshold to %u\n",
1751                         dd->ipath_pioupd_thresh);
1752                 spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags);
1753                 dd->ipath_sendctrl &= ~(INFINIPATH_S_UPDTHRESH_MASK
1754                         << INFINIPATH_S_UPDTHRESH_SHIFT);
1755                 dd->ipath_sendctrl |= dd->ipath_pioupd_thresh
1756                         << INFINIPATH_S_UPDTHRESH_SHIFT;
1757                 ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
1758                         dd->ipath_sendctrl);
1759                 spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags);
1760         }
1761 }
1762 
1763 /**
1764  * ipath_create_rcvhdrq - create a receive header queue
1765  * @dd: the infinipath device
1766  * @pd: the port data
1767  *
1768  * this must be contiguous memory (from an i/o perspective), and must be
1769  * DMA'able (which means for some systems, it will go through an IOMMU,
1770  * or be forced into a low address range).
1771  */
1772 int ipath_create_rcvhdrq(struct ipath_devdata *dd,
1773                          struct ipath_portdata *pd)
1774 {
1775         int ret = 0;
1776 
1777         if (!pd->port_rcvhdrq) {
1778                 dma_addr_t phys_hdrqtail;
1779                 gfp_t gfp_flags = GFP_USER | __GFP_COMP;
1780                 int amt = ALIGN(dd->ipath_rcvhdrcnt * dd->ipath_rcvhdrentsize *
1781                                 sizeof(u32), PAGE_SIZE);
1782 
1783                 pd->port_rcvhdrq = dma_alloc_coherent(
1784                         &dd->pcidev->dev, amt, &pd->port_rcvhdrq_phys,
1785                         gfp_flags);
1786 
1787                 if (!pd->port_rcvhdrq) {
1788                         ipath_dev_err(dd, "attempt to allocate %d bytes "
1789                                       "for port %u rcvhdrq failed\n",
1790                                       amt, pd->port_port);
1791                         ret = -ENOMEM;
1792                         goto bail;
1793                 }
1794 
1795                 if (!(dd->ipath_flags & IPATH_NODMA_RTAIL)) {
1796                         pd->port_rcvhdrtail_kvaddr = dma_alloc_coherent(
1797                                 &dd->pcidev->dev, PAGE_SIZE, &phys_hdrqtail,
1798                                 GFP_KERNEL);
1799                         if (!pd->port_rcvhdrtail_kvaddr) {
1800                                 ipath_dev_err(dd, "attempt to allocate 1 page "
1801                                         "for port %u rcvhdrqtailaddr "
1802                                         "failed\n", pd->port_port);
1803                                 ret = -ENOMEM;
1804                                 dma_free_coherent(&dd->pcidev->dev, amt,
1805                                         pd->port_rcvhdrq,
1806                                         pd->port_rcvhdrq_phys);
1807                                 pd->port_rcvhdrq = NULL;
1808                                 goto bail;
1809                         }
1810                         pd->port_rcvhdrqtailaddr_phys = phys_hdrqtail;
1811                         ipath_cdbg(VERBOSE, "port %d hdrtailaddr, %llx "
1812                                    "physical\n", pd->port_port,
1813                                    (unsigned long long) phys_hdrqtail);
1814                 }
1815 
1816                 pd->port_rcvhdrq_size = amt;
1817 
1818                 ipath_cdbg(VERBOSE, "%d pages at %p (phys %lx) size=%lu "
1819                            "for port %u rcvhdr Q\n",
1820                            amt >> PAGE_SHIFT, pd->port_rcvhdrq,
1821                            (unsigned long) pd->port_rcvhdrq_phys,
1822                            (unsigned long) pd->port_rcvhdrq_size,
1823                            pd->port_port);
1824         }
1825         else
1826                 ipath_cdbg(VERBOSE, "reuse port %d rcvhdrq @%p %llx phys; "
1827                            "hdrtailaddr@%p %llx physical\n",
1828                            pd->port_port, pd->port_rcvhdrq,
1829                            (unsigned long long) pd->port_rcvhdrq_phys,
1830                            pd->port_rcvhdrtail_kvaddr, (unsigned long long)
1831                            pd->port_rcvhdrqtailaddr_phys);
1832 
1833         /* clear for security and sanity on each use */
1834         memset(pd->port_rcvhdrq, 0, pd->port_rcvhdrq_size);
1835         if (pd->port_rcvhdrtail_kvaddr)
1836                 memset(pd->port_rcvhdrtail_kvaddr, 0, PAGE_SIZE);
1837 
1838         /*
1839          * tell chip each time we init it, even if we are re-using previous
1840          * memory (we zero the register at process close)
1841          */
1842         ipath_write_kreg_port(dd, dd->ipath_kregs->kr_rcvhdrtailaddr,
1843                               pd->port_port, pd->port_rcvhdrqtailaddr_phys);
1844         ipath_write_kreg_port(dd, dd->ipath_kregs->kr_rcvhdraddr,
1845                               pd->port_port, pd->port_rcvhdrq_phys);
1846 
1847 bail:
1848         return ret;
1849 }
1850 
1851 
1852 /*
1853  * Flush all sends that might be in the ready to send state, as well as any
1854  * that are in the process of being sent.   Used whenever we need to be
1855  * sure the send side is idle.  Cleans up all buffer state by canceling
1856  * all pio buffers, and issuing an abort, which cleans up anything in the
1857  * launch fifo.  The cancel is superfluous on some chip versions, but
1858  * it's safer to always do it.
1859  * PIOAvail bits are updated by the chip as if normal send had happened.
1860  */
1861 void ipath_cancel_sends(struct ipath_devdata *dd, int restore_sendctrl)
1862 {
1863         unsigned long flags;
1864 
1865         if (dd->ipath_flags & IPATH_IB_AUTONEG_INPROG) {
1866                 ipath_cdbg(VERBOSE, "Ignore while in autonegotiation\n");
1867                 goto bail;
1868         }
1869         /*
1870          * If we have SDMA, and it's not disabled, we have to kick off the
1871          * abort state machine, provided we aren't already aborting.
1872          * If we are in the process of aborting SDMA (!DISABLED, but ABORTING),
1873          * we skip the rest of this routine. It is already "in progress"
1874          */
1875         if (dd->ipath_flags & IPATH_HAS_SEND_DMA) {
1876                 int skip_cancel;
1877                 unsigned long *statp = &dd->ipath_sdma_status;
1878 
1879                 spin_lock_irqsave(&dd->ipath_sdma_lock, flags);
1880                 skip_cancel =
1881                         test_and_set_bit(IPATH_SDMA_ABORTING, statp)
1882                         && !test_bit(IPATH_SDMA_DISABLED, statp);
1883                 spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);
1884                 if (skip_cancel)
1885                         goto bail;
1886         }
1887 
1888         ipath_dbg("Cancelling all in-progress send buffers\n");
1889 
1890         /* skip armlaunch errs for a while */
1891         dd->ipath_lastcancel = jiffies + HZ / 2;
1892 
1893         /*
1894          * The abort bit is auto-clearing.  We also don't want pioavail
1895          * update happening during this, and we don't want any other
1896          * sends going out, so turn those off for the duration.  We read
1897          * the scratch register to be sure that cancels and the abort
1898          * have taken effect in the chip.  Otherwise two parts are same
1899          * as ipath_force_pio_avail_update()
1900          */
1901         spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags);
1902         dd->ipath_sendctrl &= ~(INFINIPATH_S_PIOBUFAVAILUPD
1903                 | INFINIPATH_S_PIOENABLE);
1904         ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
1905                 dd->ipath_sendctrl | INFINIPATH_S_ABORT);
1906         ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
1907         spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags);
1908 
1909         /* disarm all send buffers */
1910         ipath_disarm_piobufs(dd, 0,
1911                 dd->ipath_piobcnt2k + dd->ipath_piobcnt4k);
1912 
1913         if (dd->ipath_flags & IPATH_HAS_SEND_DMA)
1914                 set_bit(IPATH_SDMA_DISARMED, &dd->ipath_sdma_status);
1915 
1916         if (restore_sendctrl) {
1917                 /* else done by caller later if needed */
1918                 spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags);
1919                 dd->ipath_sendctrl |= INFINIPATH_S_PIOBUFAVAILUPD |
1920                         INFINIPATH_S_PIOENABLE;
1921                 ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
1922                         dd->ipath_sendctrl);
1923                 /* and again, be sure all have hit the chip */
1924                 ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
1925                 spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags);
1926         }
1927 
1928         if ((dd->ipath_flags & IPATH_HAS_SEND_DMA) &&
1929             !test_bit(IPATH_SDMA_DISABLED, &dd->ipath_sdma_status) &&
1930             test_bit(IPATH_SDMA_RUNNING, &dd->ipath_sdma_status)) {
1931                 spin_lock_irqsave(&dd->ipath_sdma_lock, flags);
1932                 /* only wait so long for intr */
1933                 dd->ipath_sdma_abort_intr_timeout = jiffies + HZ;
1934                 dd->ipath_sdma_reset_wait = 200;
1935                 if (!test_bit(IPATH_SDMA_SHUTDOWN, &dd->ipath_sdma_status))
1936                         tasklet_hi_schedule(&dd->ipath_sdma_abort_task);
1937                 spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);
1938         }
1939 bail:;
1940 }
1941 
1942 /*
1943  * Force an update of in-memory copy of the pioavail registers, when
1944  * needed for any of a variety of reasons.  We read the scratch register
1945  * to make it highly likely that the update will have happened by the
1946  * time we return.  If already off (as in cancel_sends above), this
1947  * routine is a nop, on the assumption that the caller will "do the
1948  * right thing".
1949  */
1950 void ipath_force_pio_avail_update(struct ipath_devdata *dd)
1951 {
1952         unsigned long flags;
1953 
1954         spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags);
1955         if (dd->ipath_sendctrl & INFINIPATH_S_PIOBUFAVAILUPD) {
1956                 ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
1957                         dd->ipath_sendctrl & ~INFINIPATH_S_PIOBUFAVAILUPD);
1958                 ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
1959                 ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
1960                         dd->ipath_sendctrl);
1961                 ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
1962         }
1963         spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags);
1964 }
1965 
1966 static void ipath_set_ib_lstate(struct ipath_devdata *dd, int linkcmd,
1967                                 int linitcmd)
1968 {
1969         u64 mod_wd;
1970         static const char *what[4] = {
1971                 [0] = "NOP",
1972                 [INFINIPATH_IBCC_LINKCMD_DOWN] = "DOWN",
1973                 [INFINIPATH_IBCC_LINKCMD_ARMED] = "ARMED",
1974                 [INFINIPATH_IBCC_LINKCMD_ACTIVE] = "ACTIVE"
1975         };
1976 
1977         if (linitcmd == INFINIPATH_IBCC_LINKINITCMD_DISABLE) {
1978                 /*
1979                  * If we are told to disable, note that so link-recovery
1980                  * code does not attempt to bring us back up.
1981                  */
1982                 preempt_disable();
1983                 dd->ipath_flags |= IPATH_IB_LINK_DISABLED;
1984                 preempt_enable();
1985         } else if (linitcmd) {
1986                 /*
1987                  * Any other linkinitcmd will lead to LINKDOWN and then
1988                  * to INIT (if all is well), so clear flag to let
1989                  * link-recovery code attempt to bring us back up.
1990                  */
1991                 preempt_disable();
1992                 dd->ipath_flags &= ~IPATH_IB_LINK_DISABLED;
1993                 preempt_enable();
1994         }
1995 
1996         mod_wd = (linkcmd << dd->ibcc_lc_shift) |
1997                 (linitcmd << INFINIPATH_IBCC_LINKINITCMD_SHIFT);
1998         ipath_cdbg(VERBOSE,
1999                 "Moving unit %u to %s (initcmd=0x%x), current ltstate is %s\n",
2000                 dd->ipath_unit, what[linkcmd], linitcmd,
2001                 ipath_ibcstatus_str[ipath_ib_linktrstate(dd,
2002                         ipath_read_kreg64(dd, dd->ipath_kregs->kr_ibcstatus))]);
2003 
2004         ipath_write_kreg(dd, dd->ipath_kregs->kr_ibcctrl,
2005                          dd->ipath_ibcctrl | mod_wd);
2006         /* read from chip so write is flushed */
2007         (void) ipath_read_kreg64(dd, dd->ipath_kregs->kr_ibcstatus);
2008 }
2009 
2010 int ipath_set_linkstate(struct ipath_devdata *dd, u8 newstate)
2011 {
2012         u32 lstate;
2013         int ret;
2014 
2015         switch (newstate) {
2016         case IPATH_IB_LINKDOWN_ONLY:
2017                 ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_DOWN, 0);
2018                 /* don't wait */
2019                 ret = 0;
2020                 goto bail;
2021 
2022         case IPATH_IB_LINKDOWN:
2023                 ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_DOWN,
2024                                         INFINIPATH_IBCC_LINKINITCMD_POLL);
2025                 /* don't wait */
2026                 ret = 0;
2027                 goto bail;
2028 
2029         case IPATH_IB_LINKDOWN_SLEEP:
2030                 ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_DOWN,
2031                                         INFINIPATH_IBCC_LINKINITCMD_SLEEP);
2032                 /* don't wait */
2033                 ret = 0;
2034                 goto bail;
2035 
2036         case IPATH_IB_LINKDOWN_DISABLE:
2037                 ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_DOWN,
2038                                         INFINIPATH_IBCC_LINKINITCMD_DISABLE);
2039                 /* don't wait */
2040                 ret = 0;
2041                 goto bail;
2042 
2043         case IPATH_IB_LINKARM:
2044                 if (dd->ipath_flags & IPATH_LINKARMED) {
2045                         ret = 0;
2046                         goto bail;
2047                 }
2048                 if (!(dd->ipath_flags &
2049                       (IPATH_LINKINIT | IPATH_LINKACTIVE))) {
2050                         ret = -EINVAL;
2051                         goto bail;
2052                 }
2053                 ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_ARMED, 0);
2054 
2055                 /*
2056                  * Since the port can transition to ACTIVE by receiving
2057                  * a non VL 15 packet, wait for either state.
2058                  */
2059                 lstate = IPATH_LINKARMED | IPATH_LINKACTIVE;
2060                 break;
2061 
2062         case IPATH_IB_LINKACTIVE:
2063                 if (dd->ipath_flags & IPATH_LINKACTIVE) {
2064                         ret = 0;
2065                         goto bail;
2066                 }
2067                 if (!(dd->ipath_flags & IPATH_LINKARMED)) {
2068                         ret = -EINVAL;
2069                         goto bail;
2070                 }
2071                 ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_ACTIVE, 0);
2072                 lstate = IPATH_LINKACTIVE;
2073                 break;
2074 
2075         case IPATH_IB_LINK_LOOPBACK:
2076                 dev_info(&dd->pcidev->dev, "Enabling IB local loopback\n");
2077                 dd->ipath_ibcctrl |= INFINIPATH_IBCC_LOOPBACK;
2078                 ipath_write_kreg(dd, dd->ipath_kregs->kr_ibcctrl,
2079                                  dd->ipath_ibcctrl);
2080 
2081                 /* turn heartbeat off, as it causes loopback to fail */
2082                 dd->ipath_f_set_ib_cfg(dd, IPATH_IB_CFG_HRTBT,
2083                                        IPATH_IB_HRTBT_OFF);
2084                 /* don't wait */
2085                 ret = 0;
2086                 goto bail;
2087 
2088         case IPATH_IB_LINK_EXTERNAL:
2089                 dev_info(&dd->pcidev->dev,
2090                         "Disabling IB local loopback (normal)\n");
2091                 dd->ipath_f_set_ib_cfg(dd, IPATH_IB_CFG_HRTBT,
2092                                        IPATH_IB_HRTBT_ON);
2093                 dd->ipath_ibcctrl &= ~INFINIPATH_IBCC_LOOPBACK;
2094                 ipath_write_kreg(dd, dd->ipath_kregs->kr_ibcctrl,
2095                                  dd->ipath_ibcctrl);
2096                 /* don't wait */
2097                 ret = 0;
2098                 goto bail;
2099 
2100         /*
2101          * Heartbeat can be explicitly enabled by the user via
2102          * "hrtbt_enable" "file", and if disabled, trying to enable here
2103          * will have no effect.  Implicit changes (heartbeat off when
2104          * loopback on, and vice versa) are included to ease testing.
2105          */
2106         case IPATH_IB_LINK_HRTBT:
2107                 ret = dd->ipath_f_set_ib_cfg(dd, IPATH_IB_CFG_HRTBT,
2108                         IPATH_IB_HRTBT_ON);
2109                 goto bail;
2110 
2111         case IPATH_IB_LINK_NO_HRTBT:
2112                 ret = dd->ipath_f_set_ib_cfg(dd, IPATH_IB_CFG_HRTBT,
2113                         IPATH_IB_HRTBT_OFF);
2114                 goto bail;
2115 
2116         default:
2117                 ipath_dbg("Invalid linkstate 0x%x requested\n", newstate);
2118                 ret = -EINVAL;
2119                 goto bail;
2120         }
2121         ret = ipath_wait_linkstate(dd, lstate, 2000);
2122 
2123 bail:
2124         return ret;
2125 }
2126 
2127 /**
2128  * ipath_set_mtu - set the MTU
2129  * @dd: the infinipath device
2130  * @arg: the new MTU
2131  *
2132  * we can handle "any" incoming size, the issue here is whether we
2133  * need to restrict our outgoing size.   For now, we don't do any
2134  * sanity checking on this, and we don't deal with what happens to
2135  * programs that are already running when the size changes.
2136  * NOTE: changing the MTU will usually cause the IBC to go back to
2137  * link INIT state...
2138  */
2139 int ipath_set_mtu(struct ipath_devdata *dd, u16 arg)
2140 {
2141         u32 piosize;
2142         int changed = 0;
2143         int ret;
2144 
2145         /*
2146          * mtu is IB data payload max.  It's the largest power of 2 less
2147          * than piosize (or even larger, since it only really controls the
2148          * largest we can receive; we can send the max of the mtu and
2149          * piosize).  We check that it's one of the valid IB sizes.
2150          */
2151         if (arg != 256 && arg != 512 && arg != 1024 && arg != 2048 &&
2152             (arg != 4096 || !ipath_mtu4096)) {
2153                 ipath_dbg("Trying to set invalid mtu %u, failing\n", arg);
2154                 ret = -EINVAL;
2155                 goto bail;
2156         }
2157         if (dd->ipath_ibmtu == arg) {
2158                 ret = 0;        /* same as current */
2159                 goto bail;
2160         }
2161 
2162         piosize = dd->ipath_ibmaxlen;
2163         dd->ipath_ibmtu = arg;
2164 
2165         if (arg >= (piosize - IPATH_PIO_MAXIBHDR)) {
2166                 /* Only if it's not the initial value (or reset to it) */
2167                 if (piosize != dd->ipath_init_ibmaxlen) {
2168                         if (arg > piosize && arg <= dd->ipath_init_ibmaxlen)
2169                                 piosize = dd->ipath_init_ibmaxlen;
2170                         dd->ipath_ibmaxlen = piosize;
2171                         changed = 1;
2172                 }
2173         } else if ((arg + IPATH_PIO_MAXIBHDR) != dd->ipath_ibmaxlen) {
2174                 piosize = arg + IPATH_PIO_MAXIBHDR;
2175                 ipath_cdbg(VERBOSE, "ibmaxlen was 0x%x, setting to 0x%x "
2176                            "(mtu 0x%x)\n", dd->ipath_ibmaxlen, piosize,
2177                            arg);
2178                 dd->ipath_ibmaxlen = piosize;
2179                 changed = 1;
2180         }
2181 
2182         if (changed) {
2183                 u64 ibc = dd->ipath_ibcctrl, ibdw;
2184                 /*
2185                  * update our housekeeping variables, and set IBC max
2186                  * size, same as init code; max IBC is max we allow in
2187                  * buffer, less the qword pbc, plus 1 for ICRC, in dwords
2188                  */
2189                 dd->ipath_ibmaxlen = piosize - 2 * sizeof(u32);
2190                 ibdw = (dd->ipath_ibmaxlen >> 2) + 1;
2191                 ibc &= ~(INFINIPATH_IBCC_MAXPKTLEN_MASK <<
2192                          dd->ibcc_mpl_shift);
2193                 ibc |= ibdw << dd->ibcc_mpl_shift;
2194                 dd->ipath_ibcctrl = ibc;
2195                 ipath_write_kreg(dd, dd->ipath_kregs->kr_ibcctrl,
2196                                  dd->ipath_ibcctrl);
2197                 dd->ipath_f_tidtemplate(dd);
2198         }
2199 
2200         ret = 0;
2201 
2202 bail:
2203         return ret;
2204 }
2205 
2206 int ipath_set_lid(struct ipath_devdata *dd, u32 lid, u8 lmc)
2207 {
2208         dd->ipath_lid = lid;
2209         dd->ipath_lmc = lmc;
2210 
2211         dd->ipath_f_set_ib_cfg(dd, IPATH_IB_CFG_LIDLMC, lid |
2212                 (~((1U << lmc) - 1)) << 16);
2213 
2214         dev_info(&dd->pcidev->dev, "We got a lid: 0x%x\n", lid);
2215 
2216         return 0;
2217 }
2218 
2219 
2220 /**
2221  * ipath_write_kreg_port - write a device's per-port 64-bit kernel register
2222  * @dd: the infinipath device
2223  * @regno: the register number to write
2224  * @port: the port containing the register
2225  * @value: the value to write
2226  *
2227  * Registers that vary with the chip implementation constants (port)
2228  * use this routine.
2229  */
2230 void ipath_write_kreg_port(const struct ipath_devdata *dd, ipath_kreg regno,
2231                           unsigned port, u64 value)
2232 {
2233         u16 where;
2234 
2235         if (port < dd->ipath_portcnt &&
2236             (regno == dd->ipath_kregs->kr_rcvhdraddr ||
2237              regno == dd->ipath_kregs->kr_rcvhdrtailaddr))
2238                 where = regno + port;
2239         else
2240                 where = -1;
2241 
2242         ipath_write_kreg(dd, where, value);
2243 }
2244 
2245 /*
2246  * Following deal with the "obviously simple" task of overriding the state
2247  * of the LEDS, which normally indicate link physical and logical status.
2248  * The complications arise in dealing with different hardware mappings
2249  * and the board-dependent routine being called from interrupts.
2250  * and then there's the requirement to _flash_ them.
2251  */
2252 #define LED_OVER_FREQ_SHIFT 8
2253 #define LED_OVER_FREQ_MASK (0xFF<<LED_OVER_FREQ_SHIFT)
2254 /* Below is "non-zero" to force override, but both actual LEDs are off */
2255 #define LED_OVER_BOTH_OFF (8)
2256 
2257 static void ipath_run_led_override(unsigned long opaque)
2258 {
2259         struct ipath_devdata *dd = (struct ipath_devdata *)opaque;
2260         int timeoff;
2261         int pidx;
2262         u64 lstate, ltstate, val;
2263 
2264         if (!(dd->ipath_flags & IPATH_INITTED))
2265                 return;
2266 
2267         pidx = dd->ipath_led_override_phase++ & 1;
2268         dd->ipath_led_override = dd->ipath_led_override_vals[pidx];
2269         timeoff = dd->ipath_led_override_timeoff;
2270 
2271         /*
2272          * below potentially restores the LED values per current status,
2273          * should also possibly setup the traffic-blink register,
2274          * but leave that to per-chip functions.
2275          */
2276         val = ipath_read_kreg64(dd, dd->ipath_kregs->kr_ibcstatus);
2277         ltstate = ipath_ib_linktrstate(dd, val);
2278         lstate = ipath_ib_linkstate(dd, val);
2279 
2280         dd->ipath_f_setextled(dd, lstate, ltstate);
2281         mod_timer(&dd->ipath_led_override_timer, jiffies + timeoff);
2282 }
2283 
2284 void ipath_set_led_override(struct ipath_devdata *dd, unsigned int val)
2285 {
2286         int timeoff, freq;
2287 
2288         if (!(dd->ipath_flags & IPATH_INITTED))
2289                 return;
2290 
2291         /* First check if we are blinking. If not, use 1HZ polling */
2292         timeoff = HZ;
2293         freq = (val & LED_OVER_FREQ_MASK) >> LED_OVER_FREQ_SHIFT;
2294 
2295         if (freq) {
2296                 /* For blink, set each phase from one nybble of val */
2297                 dd->ipath_led_override_vals[0] = val & 0xF;
2298                 dd->ipath_led_override_vals[1] = (val >> 4) & 0xF;
2299                 timeoff = (HZ << 4)/freq;
2300         } else {
2301                 /* Non-blink set both phases the same. */
2302                 dd->ipath_led_override_vals[0] = val & 0xF;
2303                 dd->ipath_led_override_vals[1] = val & 0xF;
2304         }
2305         dd->ipath_led_override_timeoff = timeoff;
2306 
2307         /*
2308          * If the timer has not already been started, do so. Use a "quick"
2309          * timeout so the function will be called soon, to look at our request.
2310          */
2311         if (atomic_inc_return(&dd->ipath_led_override_timer_active) == 1) {
2312                 /* Need to start timer */
2313                 init_timer(&dd->ipath_led_override_timer);
2314                 dd->ipath_led_override_timer.function =
2315                                                  ipath_run_led_override;
2316                 dd->ipath_led_override_timer.data = (unsigned long) dd;
2317                 dd->ipath_led_override_timer.expires = jiffies + 1;
2318                 add_timer(&dd->ipath_led_override_timer);
2319         } else
2320                 atomic_dec(&dd->ipath_led_override_timer_active);
2321 }
2322 
2323 /**
2324  * ipath_shutdown_device - shut down a device
2325  * @dd: the infinipath device
2326  *
2327  * This is called to make the device quiet when we are about to
2328  * unload the driver, and also when the device is administratively
2329  * disabled.   It does not free any data structures.
2330  * Everything it does has to be setup again by ipath_init_chip(dd,1)
2331  */
2332 void ipath_shutdown_device(struct ipath_devdata *dd)
2333 {
2334         unsigned long flags;
2335 
2336         ipath_dbg("Shutting down the device\n");
2337 
2338         ipath_hol_up(dd); /* make sure user processes aren't suspended */
2339 
2340         dd->ipath_flags |= IPATH_LINKUNK;
2341         dd->ipath_flags &= ~(IPATH_INITTED | IPATH_LINKDOWN |
2342                              IPATH_LINKINIT | IPATH_LINKARMED |
2343                              IPATH_LINKACTIVE);
2344         *dd->ipath_statusp &= ~(IPATH_STATUS_IB_CONF |
2345                                 IPATH_STATUS_IB_READY);
2346 
2347         /* mask interrupts, but not errors */
2348         ipath_write_kreg(dd, dd->ipath_kregs->kr_intmask, 0ULL);
2349 
2350         dd->ipath_rcvctrl = 0;
2351         ipath_write_kreg(dd, dd->ipath_kregs->kr_rcvctrl,
2352                          dd->ipath_rcvctrl);
2353 
2354         if (dd->ipath_flags & IPATH_HAS_SEND_DMA)
2355                 teardown_sdma(dd);
2356 
2357         /*
2358          * gracefully stop all sends allowing any in progress to trickle out
2359          * first.
2360          */
2361         spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags);
2362         dd->ipath_sendctrl = 0;
2363         ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl, dd->ipath_sendctrl);
2364         /* flush it */
2365         ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
2366         spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags);
2367 
2368         /*
2369          * enough for anything that's going to trickle out to have actually
2370          * done so.
2371          */
2372         udelay(5);
2373 
2374         dd->ipath_f_setextled(dd, 0, 0); /* make sure LEDs are off */
2375 
2376         ipath_set_ib_lstate(dd, 0, INFINIPATH_IBCC_LINKINITCMD_DISABLE);
2377         ipath_cancel_sends(dd, 0);
2378 
2379         /*
2380          * we are shutting down, so tell components that care.  We don't do
2381          * this on just a link state change, much like ethernet, a cable
2382          * unplug, etc. doesn't change driver state
2383          */
2384         signal_ib_event(dd, IB_EVENT_PORT_ERR);
2385 
2386         /* disable IBC */
2387         dd->ipath_control &= ~INFINIPATH_C_LINKENABLE;
2388         ipath_write_kreg(dd, dd->ipath_kregs->kr_control,
2389                          dd->ipath_control | INFINIPATH_C_FREEZEMODE);
2390 
2391         /*
2392          * clear SerdesEnable and turn the leds off; do this here because
2393          * we are unloading, so don't count on interrupts to move along
2394          * Turn the LEDs off explicitly for the same reason.
2395          */
2396         dd->ipath_f_quiet_serdes(dd);
2397 
2398         /* stop all the timers that might still be running */
2399         del_timer_sync(&dd->ipath_hol_timer);
2400         if (dd->ipath_stats_timer_active) {
2401                 del_timer_sync(&dd->ipath_stats_timer);
2402                 dd->ipath_stats_timer_active = 0;
2403         }
2404         if (dd->ipath_intrchk_timer.data) {
2405                 del_timer_sync(&dd->ipath_intrchk_timer);
2406                 dd->ipath_intrchk_timer.data = 0;
2407         }
2408         if (atomic_read(&dd->ipath_led_override_timer_active)) {
2409                 del_timer_sync(&dd->ipath_led_override_timer);
2410                 atomic_set(&dd->ipath_led_override_timer_active, 0);
2411         }
2412 
2413         /*
2414          * clear all interrupts and errors, so that the next time the driver
2415          * is loaded or device is enabled, we know that whatever is set
2416          * happened while we were unloaded
2417          */
2418         ipath_write_kreg(dd, dd->ipath_kregs->kr_hwerrclear,
2419                          ~0ULL & ~INFINIPATH_HWE_MEMBISTFAILED);
2420         ipath_write_kreg(dd, dd->ipath_kregs->kr_errorclear, -1LL);
2421         ipath_write_kreg(dd, dd->ipath_kregs->kr_intclear, -1LL);
2422 
2423         ipath_cdbg(VERBOSE, "Flush time and errors to EEPROM\n");
2424         ipath_update_eeprom_log(dd);
2425 }
2426 
2427 /**
2428  * ipath_free_pddata - free a port's allocated data
2429  * @dd: the infinipath device
2430  * @pd: the portdata structure
2431  *
2432  * free up any allocated data for a port
2433  * This should not touch anything that would affect a simultaneous
2434  * re-allocation of port data, because it is called after ipath_mutex
2435  * is released (and can be called from reinit as well).
2436  * It should never change any chip state, or global driver state.
2437  * (The only exception to global state is freeing the port0 port0_skbs.)
2438  */
2439 void ipath_free_pddata(struct ipath_devdata *dd, struct ipath_portdata *pd)
2440 {
2441         if (!pd)
2442                 return;
2443 
2444         if (pd->port_rcvhdrq) {
2445                 ipath_cdbg(VERBOSE, "free closed port %d rcvhdrq @ %p "
2446                            "(size=%lu)\n", pd->port_port, pd->port_rcvhdrq,
2447                            (unsigned long) pd->port_rcvhdrq_size);
2448                 dma_free_coherent(&dd->pcidev->dev, pd->port_rcvhdrq_size,
2449                                   pd->port_rcvhdrq, pd->port_rcvhdrq_phys);
2450                 pd->port_rcvhdrq = NULL;
2451                 if (pd->port_rcvhdrtail_kvaddr) {
2452                         dma_free_coherent(&dd->pcidev->dev, PAGE_SIZE,
2453                                          pd->port_rcvhdrtail_kvaddr,
2454                                          pd->port_rcvhdrqtailaddr_phys);
2455                         pd->port_rcvhdrtail_kvaddr = NULL;
2456                 }
2457         }
2458         if (pd->port_port && pd->port_rcvegrbuf) {
2459                 unsigned e;
2460 
2461                 for (e = 0; e < pd->port_rcvegrbuf_chunks; e++) {
2462                         void *base = pd->port_rcvegrbuf[e];
2463                         size_t size = pd->port_rcvegrbuf_size;
2464 
2465                         ipath_cdbg(VERBOSE, "egrbuf free(%p, %lu), "
2466                                    "chunk %u/%u\n", base,
2467                                    (unsigned long) size,
2468                                    e, pd->port_rcvegrbuf_chunks);
2469                         dma_free_coherent(&dd->pcidev->dev, size,
2470                                 base, pd->port_rcvegrbuf_phys[e]);
2471                 }
2472                 kfree(pd->port_rcvegrbuf);
2473                 pd->port_rcvegrbuf = NULL;
2474                 kfree(pd->port_rcvegrbuf_phys);
2475                 pd->port_rcvegrbuf_phys = NULL;
2476                 pd->port_rcvegrbuf_chunks = 0;
2477         } else if (pd->port_port == 0 && dd->ipath_port0_skbinfo) {
2478                 unsigned e;
2479                 struct ipath_skbinfo *skbinfo = dd->ipath_port0_skbinfo;
2480 
2481                 dd->ipath_port0_skbinfo = NULL;
2482                 ipath_cdbg(VERBOSE, "free closed port %d "
2483                            "ipath_port0_skbinfo @ %p\n", pd->port_port,
2484                            skbinfo);
2485                 for (e = 0; e < dd->ipath_p0_rcvegrcnt; e++)
2486                         if (skbinfo[e].skb) {
2487                                 pci_unmap_single(dd->pcidev, skbinfo[e].phys,
2488                                                  dd->ipath_ibmaxlen,
2489                                                  PCI_DMA_FROMDEVICE);
2490                                 dev_kfree_skb(skbinfo[e].skb);
2491                         }
2492                 vfree(skbinfo);
2493         }
2494         kfree(pd->port_tid_pg_list);
2495         vfree(pd->subport_uregbase);
2496         vfree(pd->subport_rcvegrbuf);
2497         vfree(pd->subport_rcvhdr_base);
2498         kfree(pd);
2499 }
2500 
2501 static int __init infinipath_init(void)
2502 {
2503         int ret;
2504 
2505         if (ipath_debug & __IPATH_DBG)
2506                 printk(KERN_INFO DRIVER_LOAD_MSG "%s", ib_ipath_version);
2507 
2508         /*
2509          * These must be called before the driver is registered with
2510          * the PCI subsystem.
2511          */
2512         idr_init(&unit_table);
2513 
2514         ret = pci_register_driver(&ipath_driver);
2515         if (ret < 0) {
2516                 printk(KERN_ERR IPATH_DRV_NAME
2517                        ": Unable to register driver: error %d\n", -ret);
2518                 goto bail_unit;
2519         }
2520 
2521         ret = ipath_init_ipathfs();
2522         if (ret < 0) {
2523                 printk(KERN_ERR IPATH_DRV_NAME ": Unable to create "
2524                        "ipathfs: error %d\n", -ret);
2525                 goto bail_pci;
2526         }
2527 
2528         goto bail;
2529 
2530 bail_pci:
2531         pci_unregister_driver(&ipath_driver);
2532 
2533 bail_unit:
2534         idr_destroy(&unit_table);
2535 
2536 bail:
2537         return ret;
2538 }
2539 
2540 static void __exit infinipath_cleanup(void)
2541 {
2542         ipath_exit_ipathfs();
2543 
2544         ipath_cdbg(VERBOSE, "Unregistering pci driver\n");
2545         pci_unregister_driver(&ipath_driver);
2546 
2547         idr_destroy(&unit_table);
2548 }
2549 
2550 /**
2551  * ipath_reset_device - reset the chip if possible
2552  * @unit: the device to reset
2553  *
2554  * Whether or not reset is successful, we attempt to re-initialize the chip
2555  * (that is, much like a driver unload/reload).  We clear the INITTED flag
2556  * so that the various entry points will fail until we reinitialize.  For
2557  * now, we only allow this if no user ports are open that use chip resources
2558  */
2559 int ipath_reset_device(int unit)
2560 {
2561         int ret, i;
2562         struct ipath_devdata *dd = ipath_lookup(unit);
2563         unsigned long flags;
2564 
2565         if (!dd) {
2566                 ret = -ENODEV;
2567                 goto bail;
2568         }
2569 
2570         if (atomic_read(&dd->ipath_led_override_timer_active)) {
2571                 /* Need to stop LED timer, _then_ shut off LEDs */
2572                 del_timer_sync(&dd->ipath_led_override_timer);
2573                 atomic_set(&dd->ipath_led_override_timer_active, 0);
2574         }
2575 
2576         /* Shut off LEDs after we are sure timer is not running */
2577         dd->ipath_led_override = LED_OVER_BOTH_OFF;
2578         dd->ipath_f_setextled(dd, 0, 0);
2579 
2580         dev_info(&dd->pcidev->dev, "Reset on unit %u requested\n", unit);
2581 
2582         if (!dd->ipath_kregbase || !(dd->ipath_flags & IPATH_PRESENT)) {
2583                 dev_info(&dd->pcidev->dev, "Invalid unit number %u or "
2584                          "not initialized or not present\n", unit);
2585                 ret = -ENXIO;
2586                 goto bail;
2587         }
2588 
2589         spin_lock_irqsave(&dd->ipath_uctxt_lock, flags);
2590         if (dd->ipath_pd)
2591                 for (i = 1; i < dd->ipath_cfgports; i++) {
2592                         if (!dd->ipath_pd[i] || !dd->ipath_pd[i]->port_cnt)
2593                                 continue;
2594                         spin_unlock_irqrestore(&dd->ipath_uctxt_lock, flags);
2595                         ipath_dbg("unit %u port %d is in use "
2596                                   "(PID %u cmd %s), can't reset\n",
2597                                   unit, i,
2598                                   pid_nr(dd->ipath_pd[i]->port_pid),
2599                                   dd->ipath_pd[i]->port_comm);
2600                         ret = -EBUSY;
2601                         goto bail;
2602                 }
2603         spin_unlock_irqrestore(&dd->ipath_uctxt_lock, flags);
2604 
2605         if (dd->ipath_flags & IPATH_HAS_SEND_DMA)
2606                 teardown_sdma(dd);
2607 
2608         dd->ipath_flags &= ~IPATH_INITTED;
2609         ipath_write_kreg(dd, dd->ipath_kregs->kr_intmask, 0ULL);
2610         ret = dd->ipath_f_reset(dd);
2611         if (ret == 1) {
2612                 ipath_dbg("Reinitializing unit %u after reset attempt\n",
2613                           unit);
2614                 ret = ipath_init_chip(dd, 1);
2615         } else
2616                 ret = -EAGAIN;
2617         if (ret)
2618                 ipath_dev_err(dd, "Reinitialize unit %u after "
2619                               "reset failed with %d\n", unit, ret);
2620         else
2621                 dev_info(&dd->pcidev->dev, "Reinitialized unit %u after "
2622                          "resetting\n", unit);
2623 
2624 bail:
2625         return ret;
2626 }
2627 
2628 /*
2629  * send a signal to all the processes that have the driver open
2630  * through the normal interfaces (i.e., everything other than diags
2631  * interface).  Returns number of signalled processes.
2632  */
2633 static int ipath_signal_procs(struct ipath_devdata *dd, int sig)
2634 {
2635         int i, sub, any = 0;
2636         struct pid *pid;
2637         unsigned long flags;
2638 
2639         if (!dd->ipath_pd)
2640                 return 0;
2641 
2642         spin_lock_irqsave(&dd->ipath_uctxt_lock, flags);
2643         for (i = 1; i < dd->ipath_cfgports; i++) {
2644                 if (!dd->ipath_pd[i] || !dd->ipath_pd[i]->port_cnt)
2645                         continue;
2646                 pid = dd->ipath_pd[i]->port_pid;
2647                 if (!pid)
2648                         continue;
2649 
2650                 dev_info(&dd->pcidev->dev, "context %d in use "
2651                           "(PID %u), sending signal %d\n",
2652                           i, pid_nr(pid), sig);
2653                 kill_pid(pid, sig, 1);
2654                 any++;
2655                 for (sub = 0; sub < INFINIPATH_MAX_SUBPORT; sub++) {
2656                         pid = dd->ipath_pd[i]->port_subpid[sub];
2657                         if (!pid)
2658                                 continue;
2659                         dev_info(&dd->pcidev->dev, "sub-context "
2660                                 "%d:%d in use (PID %u), sending "
2661                                 "signal %d\n", i, sub, pid_nr(pid), sig);
2662                         kill_pid(pid, sig, 1);
2663                         any++;
2664                 }
2665         }
2666         spin_unlock_irqrestore(&dd->ipath_uctxt_lock, flags);
2667         return any;
2668 }
2669 
2670 static void ipath_hol_signal_down(struct ipath_devdata *dd)
2671 {
2672         if (ipath_signal_procs(dd, SIGSTOP))
2673                 ipath_dbg("Stopped some processes\n");
2674         ipath_cancel_sends(dd, 1);
2675 }
2676 
2677 
2678 static void ipath_hol_signal_up(struct ipath_devdata *dd)
2679 {
2680         if (ipath_signal_procs(dd, SIGCONT))
2681                 ipath_dbg("Continued some processes\n");
2682 }
2683 
2684 /*
2685  * link is down, stop any users processes, and flush pending sends
2686  * to prevent HoL blocking, then start the HoL timer that
2687  * periodically continues, then stop procs, so they can detect
2688  * link down if they want, and do something about it.
2689  * Timer may already be running, so use mod_timer, not add_timer.
2690  */
2691 void ipath_hol_down(struct ipath_devdata *dd)
2692 {
2693         dd->ipath_hol_state = IPATH_HOL_DOWN;
2694         ipath_hol_signal_down(dd);
2695         dd->ipath_hol_next = IPATH_HOL_DOWNCONT;
2696         dd->ipath_hol_timer.expires = jiffies +
2697                 msecs_to_jiffies(ipath_hol_timeout_ms);
2698         mod_timer(&dd->ipath_hol_timer, dd->ipath_hol_timer.expires);
2699 }
2700 
2701 /*
2702  * link is up, continue any user processes, and ensure timer
2703  * is a nop, if running.  Let timer keep running, if set; it
2704  * will nop when it sees the link is up
2705  */
2706 void ipath_hol_up(struct ipath_devdata *dd)
2707 {
2708         ipath_hol_signal_up(dd);
2709         dd->ipath_hol_state = IPATH_HOL_UP;
2710 }
2711 
2712 /*
2713  * toggle the running/not running state of user proceses
2714  * to prevent HoL blocking on chip resources, but still allow
2715  * user processes to do link down special case handling.
2716  * Should only be called via the timer
2717  */
2718 void ipath_hol_event(unsigned long opaque)
2719 {
2720         struct ipath_devdata *dd = (struct ipath_devdata *)opaque;
2721 
2722         if (dd->ipath_hol_next == IPATH_HOL_DOWNSTOP
2723                 && dd->ipath_hol_state != IPATH_HOL_UP) {
2724                 dd->ipath_hol_next = IPATH_HOL_DOWNCONT;
2725                 ipath_dbg("Stopping processes\n");
2726                 ipath_hol_signal_down(dd);
2727         } else { /* may do "extra" if also in ipath_hol_up() */
2728                 dd->ipath_hol_next = IPATH_HOL_DOWNSTOP;
2729                 ipath_dbg("Continuing processes\n");
2730                 ipath_hol_signal_up(dd);
2731         }
2732         if (dd->ipath_hol_state == IPATH_HOL_UP)
2733                 ipath_dbg("link's up, don't resched timer\n");
2734         else {
2735                 dd->ipath_hol_timer.expires = jiffies +
2736                         msecs_to_jiffies(ipath_hol_timeout_ms);
2737                 mod_timer(&dd->ipath_hol_timer,
2738                         dd->ipath_hol_timer.expires);
2739         }
2740 }
2741 
2742 int ipath_set_rx_pol_inv(struct ipath_devdata *dd, u8 new_pol_inv)
2743 {
2744         u64 val;
2745 
2746         if (new_pol_inv > INFINIPATH_XGXS_RX_POL_MASK)
2747                 return -1;
2748         if (dd->ipath_rx_pol_inv != new_pol_inv) {
2749                 dd->ipath_rx_pol_inv = new_pol_inv;
2750                 val = ipath_read_kreg64(dd, dd->ipath_kregs->kr_xgxsconfig);
2751                 val &= ~(INFINIPATH_XGXS_RX_POL_MASK <<
2752                          INFINIPATH_XGXS_RX_POL_SHIFT);
2753                 val |= ((u64)dd->ipath_rx_pol_inv) <<
2754                         INFINIPATH_XGXS_RX_POL_SHIFT;
2755                 ipath_write_kreg(dd, dd->ipath_kregs->kr_xgxsconfig, val);
2756         }
2757         return 0;
2758 }
2759 
2760 /*
2761  * Disable and enable the armlaunch error.  Used for PIO bandwidth testing on
2762  * the 7220, which is count-based, rather than trigger-based.  Safe for the
2763  * driver check, since it's at init.   Not completely safe when used for
2764  * user-mode checking, since some error checking can be lost, but not
2765  * particularly risky, and only has problematic side-effects in the face of
2766  * very buggy user code.  There is no reference counting, but that's also
2767  * fine, given the intended use.
2768  */
2769 void ipath_enable_armlaunch(struct ipath_devdata *dd)
2770 {
2771         dd->ipath_lasterror &= ~INFINIPATH_E_SPIOARMLAUNCH;
2772         ipath_write_kreg(dd, dd->ipath_kregs->kr_errorclear,
2773                 INFINIPATH_E_SPIOARMLAUNCH);
2774         dd->ipath_errormask |= INFINIPATH_E_SPIOARMLAUNCH;
2775         ipath_write_kreg(dd, dd->ipath_kregs->kr_errormask,
2776                 dd->ipath_errormask);
2777 }
2778 
2779 void ipath_disable_armlaunch(struct ipath_devdata *dd)
2780 {
2781         /* so don't re-enable if already set */
2782         dd->ipath_maskederrs &= ~INFINIPATH_E_SPIOARMLAUNCH;
2783         dd->ipath_errormask &= ~INFINIPATH_E_SPIOARMLAUNCH;
2784         ipath_write_kreg(dd, dd->ipath_kregs->kr_errormask,
2785                 dd->ipath_errormask);
2786 }
2787 
2788 module_init(infinipath_init);
2789 module_exit(infinipath_cleanup);
2790 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us