Version:  2.0.40 2.2.26 2.4.37 2.6.39 3.0 3.1 3.2 3.3 3.4 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14 3.15

Linux/drivers/virtio/virtio_pci.c

  1 /*
  2  * Virtio PCI driver
  3  *
  4  * This module allows virtio devices to be used over a virtual PCI device.
  5  * This can be used with QEMU based VMMs like KVM or Xen.
  6  *
  7  * Copyright IBM Corp. 2007
  8  *
  9  * Authors:
 10  *  Anthony Liguori  <aliguori@us.ibm.com>
 11  *
 12  * This work is licensed under the terms of the GNU GPL, version 2 or later.
 13  * See the COPYING file in the top-level directory.
 14  *
 15  */
 16 
 17 #include <linux/module.h>
 18 #include <linux/list.h>
 19 #include <linux/pci.h>
 20 #include <linux/slab.h>
 21 #include <linux/interrupt.h>
 22 #include <linux/virtio.h>
 23 #include <linux/virtio_config.h>
 24 #include <linux/virtio_ring.h>
 25 #include <linux/virtio_pci.h>
 26 #include <linux/highmem.h>
 27 #include <linux/spinlock.h>
 28 
 29 MODULE_AUTHOR("Anthony Liguori <aliguori@us.ibm.com>");
 30 MODULE_DESCRIPTION("virtio-pci");
 31 MODULE_LICENSE("GPL");
 32 MODULE_VERSION("1");
 33 
 34 /* Our device structure */
 35 struct virtio_pci_device
 36 {
 37         struct virtio_device vdev;
 38         struct pci_dev *pci_dev;
 39 
 40         /* the IO mapping for the PCI config space */
 41         void __iomem *ioaddr;
 42 
 43         /* a list of queues so we can dispatch IRQs */
 44         spinlock_t lock;
 45         struct list_head virtqueues;
 46 
 47         /* MSI-X support */
 48         int msix_enabled;
 49         int intx_enabled;
 50         struct msix_entry *msix_entries;
 51         cpumask_var_t *msix_affinity_masks;
 52         /* Name strings for interrupts. This size should be enough,
 53          * and I'm too lazy to allocate each name separately. */
 54         char (*msix_names)[256];
 55         /* Number of available vectors */
 56         unsigned msix_vectors;
 57         /* Vectors allocated, excluding per-vq vectors if any */
 58         unsigned msix_used_vectors;
 59 
 60         /* Status saved during hibernate/restore */
 61         u8 saved_status;
 62 
 63         /* Whether we have vector per vq */
 64         bool per_vq_vectors;
 65 };
 66 
 67 /* Constants for MSI-X */
 68 /* Use first vector for configuration changes, second and the rest for
 69  * virtqueues Thus, we need at least 2 vectors for MSI. */
 70 enum {
 71         VP_MSIX_CONFIG_VECTOR = 0,
 72         VP_MSIX_VQ_VECTOR = 1,
 73 };
 74 
 75 struct virtio_pci_vq_info
 76 {
 77         /* the actual virtqueue */
 78         struct virtqueue *vq;
 79 
 80         /* the number of entries in the queue */
 81         int num;
 82 
 83         /* the virtual address of the ring queue */
 84         void *queue;
 85 
 86         /* the list node for the virtqueues list */
 87         struct list_head node;
 88 
 89         /* MSI-X vector (or none) */
 90         unsigned msix_vector;
 91 };
 92 
 93 /* Qumranet donated their vendor ID for devices 0x1000 thru 0x10FF. */
 94 static DEFINE_PCI_DEVICE_TABLE(virtio_pci_id_table) = {
 95         { PCI_DEVICE(0x1af4, PCI_ANY_ID) },
 96         { 0 }
 97 };
 98 
 99 MODULE_DEVICE_TABLE(pci, virtio_pci_id_table);
100 
101 /* Convert a generic virtio device to our structure */
102 static struct virtio_pci_device *to_vp_device(struct virtio_device *vdev)
103 {
104         return container_of(vdev, struct virtio_pci_device, vdev);
105 }
106 
107 /* virtio config->get_features() implementation */
108 static u32 vp_get_features(struct virtio_device *vdev)
109 {
110         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
111 
112         /* When someone needs more than 32 feature bits, we'll need to
113          * steal a bit to indicate that the rest are somewhere else. */
114         return ioread32(vp_dev->ioaddr + VIRTIO_PCI_HOST_FEATURES);
115 }
116 
117 /* virtio config->finalize_features() implementation */
118 static void vp_finalize_features(struct virtio_device *vdev)
119 {
120         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
121 
122         /* Give virtio_ring a chance to accept features. */
123         vring_transport_features(vdev);
124 
125         /* We only support 32 feature bits. */
126         BUILD_BUG_ON(ARRAY_SIZE(vdev->features) != 1);
127         iowrite32(vdev->features[0], vp_dev->ioaddr+VIRTIO_PCI_GUEST_FEATURES);
128 }
129 
130 /* virtio config->get() implementation */
131 static void vp_get(struct virtio_device *vdev, unsigned offset,
132                    void *buf, unsigned len)
133 {
134         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
135         void __iomem *ioaddr = vp_dev->ioaddr +
136                                 VIRTIO_PCI_CONFIG(vp_dev) + offset;
137         u8 *ptr = buf;
138         int i;
139 
140         for (i = 0; i < len; i++)
141                 ptr[i] = ioread8(ioaddr + i);
142 }
143 
144 /* the config->set() implementation.  it's symmetric to the config->get()
145  * implementation */
146 static void vp_set(struct virtio_device *vdev, unsigned offset,
147                    const void *buf, unsigned len)
148 {
149         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
150         void __iomem *ioaddr = vp_dev->ioaddr +
151                                 VIRTIO_PCI_CONFIG(vp_dev) + offset;
152         const u8 *ptr = buf;
153         int i;
154 
155         for (i = 0; i < len; i++)
156                 iowrite8(ptr[i], ioaddr + i);
157 }
158 
159 /* config->{get,set}_status() implementations */
160 static u8 vp_get_status(struct virtio_device *vdev)
161 {
162         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
163         return ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
164 }
165 
166 static void vp_set_status(struct virtio_device *vdev, u8 status)
167 {
168         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
169         /* We should never be setting status to 0. */
170         BUG_ON(status == 0);
171         iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
172 }
173 
174 /* wait for pending irq handlers */
175 static void vp_synchronize_vectors(struct virtio_device *vdev)
176 {
177         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
178         int i;
179 
180         if (vp_dev->intx_enabled)
181                 synchronize_irq(vp_dev->pci_dev->irq);
182 
183         for (i = 0; i < vp_dev->msix_vectors; ++i)
184                 synchronize_irq(vp_dev->msix_entries[i].vector);
185 }
186 
187 static void vp_reset(struct virtio_device *vdev)
188 {
189         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
190         /* 0 status means a reset. */
191         iowrite8(0, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
192         /* Flush out the status write, and flush in device writes,
193          * including MSi-X interrupts, if any. */
194         ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
195         /* Flush pending VQ/configuration callbacks. */
196         vp_synchronize_vectors(vdev);
197 }
198 
199 /* the notify function used when creating a virt queue */
200 static bool vp_notify(struct virtqueue *vq)
201 {
202         struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
203 
204         /* we write the queue's selector into the notification register to
205          * signal the other end */
206         iowrite16(vq->index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_NOTIFY);
207         return true;
208 }
209 
210 /* Handle a configuration change: Tell driver if it wants to know. */
211 static irqreturn_t vp_config_changed(int irq, void *opaque)
212 {
213         struct virtio_pci_device *vp_dev = opaque;
214         struct virtio_driver *drv;
215         drv = container_of(vp_dev->vdev.dev.driver,
216                            struct virtio_driver, driver);
217 
218         if (drv && drv->config_changed)
219                 drv->config_changed(&vp_dev->vdev);
220         return IRQ_HANDLED;
221 }
222 
223 /* Notify all virtqueues on an interrupt. */
224 static irqreturn_t vp_vring_interrupt(int irq, void *opaque)
225 {
226         struct virtio_pci_device *vp_dev = opaque;
227         struct virtio_pci_vq_info *info;
228         irqreturn_t ret = IRQ_NONE;
229         unsigned long flags;
230 
231         spin_lock_irqsave(&vp_dev->lock, flags);
232         list_for_each_entry(info, &vp_dev->virtqueues, node) {
233                 if (vring_interrupt(irq, info->vq) == IRQ_HANDLED)
234                         ret = IRQ_HANDLED;
235         }
236         spin_unlock_irqrestore(&vp_dev->lock, flags);
237 
238         return ret;
239 }
240 
241 /* A small wrapper to also acknowledge the interrupt when it's handled.
242  * I really need an EIO hook for the vring so I can ack the interrupt once we
243  * know that we'll be handling the IRQ but before we invoke the callback since
244  * the callback may notify the host which results in the host attempting to
245  * raise an interrupt that we would then mask once we acknowledged the
246  * interrupt. */
247 static irqreturn_t vp_interrupt(int irq, void *opaque)
248 {
249         struct virtio_pci_device *vp_dev = opaque;
250         u8 isr;
251 
252         /* reading the ISR has the effect of also clearing it so it's very
253          * important to save off the value. */
254         isr = ioread8(vp_dev->ioaddr + VIRTIO_PCI_ISR);
255 
256         /* It's definitely not us if the ISR was not high */
257         if (!isr)
258                 return IRQ_NONE;
259 
260         /* Configuration change?  Tell driver if it wants to know. */
261         if (isr & VIRTIO_PCI_ISR_CONFIG)
262                 vp_config_changed(irq, opaque);
263 
264         return vp_vring_interrupt(irq, opaque);
265 }
266 
267 static void vp_free_vectors(struct virtio_device *vdev)
268 {
269         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
270         int i;
271 
272         if (vp_dev->intx_enabled) {
273                 free_irq(vp_dev->pci_dev->irq, vp_dev);
274                 vp_dev->intx_enabled = 0;
275         }
276 
277         for (i = 0; i < vp_dev->msix_used_vectors; ++i)
278                 free_irq(vp_dev->msix_entries[i].vector, vp_dev);
279 
280         for (i = 0; i < vp_dev->msix_vectors; i++)
281                 if (vp_dev->msix_affinity_masks[i])
282                         free_cpumask_var(vp_dev->msix_affinity_masks[i]);
283 
284         if (vp_dev->msix_enabled) {
285                 /* Disable the vector used for configuration */
286                 iowrite16(VIRTIO_MSI_NO_VECTOR,
287                           vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR);
288                 /* Flush the write out to device */
289                 ioread16(vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR);
290 
291                 pci_disable_msix(vp_dev->pci_dev);
292                 vp_dev->msix_enabled = 0;
293         }
294 
295         vp_dev->msix_vectors = 0;
296         vp_dev->msix_used_vectors = 0;
297         kfree(vp_dev->msix_names);
298         vp_dev->msix_names = NULL;
299         kfree(vp_dev->msix_entries);
300         vp_dev->msix_entries = NULL;
301         kfree(vp_dev->msix_affinity_masks);
302         vp_dev->msix_affinity_masks = NULL;
303 }
304 
305 static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors,
306                                    bool per_vq_vectors)
307 {
308         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
309         const char *name = dev_name(&vp_dev->vdev.dev);
310         unsigned i, v;
311         int err = -ENOMEM;
312 
313         vp_dev->msix_vectors = nvectors;
314 
315         vp_dev->msix_entries = kmalloc(nvectors * sizeof *vp_dev->msix_entries,
316                                        GFP_KERNEL);
317         if (!vp_dev->msix_entries)
318                 goto error;
319         vp_dev->msix_names = kmalloc(nvectors * sizeof *vp_dev->msix_names,
320                                      GFP_KERNEL);
321         if (!vp_dev->msix_names)
322                 goto error;
323         vp_dev->msix_affinity_masks
324                 = kzalloc(nvectors * sizeof *vp_dev->msix_affinity_masks,
325                           GFP_KERNEL);
326         if (!vp_dev->msix_affinity_masks)
327                 goto error;
328         for (i = 0; i < nvectors; ++i)
329                 if (!alloc_cpumask_var(&vp_dev->msix_affinity_masks[i],
330                                         GFP_KERNEL))
331                         goto error;
332 
333         for (i = 0; i < nvectors; ++i)
334                 vp_dev->msix_entries[i].entry = i;
335 
336         err = pci_enable_msix_exact(vp_dev->pci_dev,
337                                     vp_dev->msix_entries, nvectors);
338         if (err)
339                 goto error;
340         vp_dev->msix_enabled = 1;
341 
342         /* Set the vector used for configuration */
343         v = vp_dev->msix_used_vectors;
344         snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
345                  "%s-config", name);
346         err = request_irq(vp_dev->msix_entries[v].vector,
347                           vp_config_changed, 0, vp_dev->msix_names[v],
348                           vp_dev);
349         if (err)
350                 goto error;
351         ++vp_dev->msix_used_vectors;
352 
353         iowrite16(v, vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR);
354         /* Verify we had enough resources to assign the vector */
355         v = ioread16(vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR);
356         if (v == VIRTIO_MSI_NO_VECTOR) {
357                 err = -EBUSY;
358                 goto error;
359         }
360 
361         if (!per_vq_vectors) {
362                 /* Shared vector for all VQs */
363                 v = vp_dev->msix_used_vectors;
364                 snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
365                          "%s-virtqueues", name);
366                 err = request_irq(vp_dev->msix_entries[v].vector,
367                                   vp_vring_interrupt, 0, vp_dev->msix_names[v],
368                                   vp_dev);
369                 if (err)
370                         goto error;
371                 ++vp_dev->msix_used_vectors;
372         }
373         return 0;
374 error:
375         vp_free_vectors(vdev);
376         return err;
377 }
378 
379 static int vp_request_intx(struct virtio_device *vdev)
380 {
381         int err;
382         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
383 
384         err = request_irq(vp_dev->pci_dev->irq, vp_interrupt,
385                           IRQF_SHARED, dev_name(&vdev->dev), vp_dev);
386         if (!err)
387                 vp_dev->intx_enabled = 1;
388         return err;
389 }
390 
391 static struct virtqueue *setup_vq(struct virtio_device *vdev, unsigned index,
392                                   void (*callback)(struct virtqueue *vq),
393                                   const char *name,
394                                   u16 msix_vec)
395 {
396         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
397         struct virtio_pci_vq_info *info;
398         struct virtqueue *vq;
399         unsigned long flags, size;
400         u16 num;
401         int err;
402 
403         /* Select the queue we're interested in */
404         iowrite16(index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_SEL);
405 
406         /* Check if queue is either not available or already active. */
407         num = ioread16(vp_dev->ioaddr + VIRTIO_PCI_QUEUE_NUM);
408         if (!num || ioread32(vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN))
409                 return ERR_PTR(-ENOENT);
410 
411         /* allocate and fill out our structure the represents an active
412          * queue */
413         info = kmalloc(sizeof(struct virtio_pci_vq_info), GFP_KERNEL);
414         if (!info)
415                 return ERR_PTR(-ENOMEM);
416 
417         info->num = num;
418         info->msix_vector = msix_vec;
419 
420         size = PAGE_ALIGN(vring_size(num, VIRTIO_PCI_VRING_ALIGN));
421         info->queue = alloc_pages_exact(size, GFP_KERNEL|__GFP_ZERO);
422         if (info->queue == NULL) {
423                 err = -ENOMEM;
424                 goto out_info;
425         }
426 
427         /* activate the queue */
428         iowrite32(virt_to_phys(info->queue) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT,
429                   vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
430 
431         /* create the vring */
432         vq = vring_new_virtqueue(index, info->num, VIRTIO_PCI_VRING_ALIGN, vdev,
433                                  true, info->queue, vp_notify, callback, name);
434         if (!vq) {
435                 err = -ENOMEM;
436                 goto out_activate_queue;
437         }
438 
439         vq->priv = info;
440         info->vq = vq;
441 
442         if (msix_vec != VIRTIO_MSI_NO_VECTOR) {
443                 iowrite16(msix_vec, vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR);
444                 msix_vec = ioread16(vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR);
445                 if (msix_vec == VIRTIO_MSI_NO_VECTOR) {
446                         err = -EBUSY;
447                         goto out_assign;
448                 }
449         }
450 
451         if (callback) {
452                 spin_lock_irqsave(&vp_dev->lock, flags);
453                 list_add(&info->node, &vp_dev->virtqueues);
454                 spin_unlock_irqrestore(&vp_dev->lock, flags);
455         } else {
456                 INIT_LIST_HEAD(&info->node);
457         }
458 
459         return vq;
460 
461 out_assign:
462         vring_del_virtqueue(vq);
463 out_activate_queue:
464         iowrite32(0, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
465         free_pages_exact(info->queue, size);
466 out_info:
467         kfree(info);
468         return ERR_PTR(err);
469 }
470 
471 static void vp_del_vq(struct virtqueue *vq)
472 {
473         struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
474         struct virtio_pci_vq_info *info = vq->priv;
475         unsigned long flags, size;
476 
477         spin_lock_irqsave(&vp_dev->lock, flags);
478         list_del(&info->node);
479         spin_unlock_irqrestore(&vp_dev->lock, flags);
480 
481         iowrite16(vq->index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_SEL);
482 
483         if (vp_dev->msix_enabled) {
484                 iowrite16(VIRTIO_MSI_NO_VECTOR,
485                           vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR);
486                 /* Flush the write out to device */
487                 ioread8(vp_dev->ioaddr + VIRTIO_PCI_ISR);
488         }
489 
490         vring_del_virtqueue(vq);
491 
492         /* Select and deactivate the queue */
493         iowrite32(0, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
494 
495         size = PAGE_ALIGN(vring_size(info->num, VIRTIO_PCI_VRING_ALIGN));
496         free_pages_exact(info->queue, size);
497         kfree(info);
498 }
499 
500 /* the config->del_vqs() implementation */
501 static void vp_del_vqs(struct virtio_device *vdev)
502 {
503         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
504         struct virtqueue *vq, *n;
505         struct virtio_pci_vq_info *info;
506 
507         list_for_each_entry_safe(vq, n, &vdev->vqs, list) {
508                 info = vq->priv;
509                 if (vp_dev->per_vq_vectors &&
510                         info->msix_vector != VIRTIO_MSI_NO_VECTOR)
511                         free_irq(vp_dev->msix_entries[info->msix_vector].vector,
512                                  vq);
513                 vp_del_vq(vq);
514         }
515         vp_dev->per_vq_vectors = false;
516 
517         vp_free_vectors(vdev);
518 }
519 
520 static int vp_try_to_find_vqs(struct virtio_device *vdev, unsigned nvqs,
521                               struct virtqueue *vqs[],
522                               vq_callback_t *callbacks[],
523                               const char *names[],
524                               bool use_msix,
525                               bool per_vq_vectors)
526 {
527         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
528         u16 msix_vec;
529         int i, err, nvectors, allocated_vectors;
530 
531         if (!use_msix) {
532                 /* Old style: one normal interrupt for change and all vqs. */
533                 err = vp_request_intx(vdev);
534                 if (err)
535                         goto error_request;
536         } else {
537                 if (per_vq_vectors) {
538                         /* Best option: one for change interrupt, one per vq. */
539                         nvectors = 1;
540                         for (i = 0; i < nvqs; ++i)
541                                 if (callbacks[i])
542                                         ++nvectors;
543                 } else {
544                         /* Second best: one for change, shared for all vqs. */
545                         nvectors = 2;
546                 }
547 
548                 err = vp_request_msix_vectors(vdev, nvectors, per_vq_vectors);
549                 if (err)
550                         goto error_request;
551         }
552 
553         vp_dev->per_vq_vectors = per_vq_vectors;
554         allocated_vectors = vp_dev->msix_used_vectors;
555         for (i = 0; i < nvqs; ++i) {
556                 if (!names[i]) {
557                         vqs[i] = NULL;
558                         continue;
559                 } else if (!callbacks[i] || !vp_dev->msix_enabled)
560                         msix_vec = VIRTIO_MSI_NO_VECTOR;
561                 else if (vp_dev->per_vq_vectors)
562                         msix_vec = allocated_vectors++;
563                 else
564                         msix_vec = VP_MSIX_VQ_VECTOR;
565                 vqs[i] = setup_vq(vdev, i, callbacks[i], names[i], msix_vec);
566                 if (IS_ERR(vqs[i])) {
567                         err = PTR_ERR(vqs[i]);
568                         goto error_find;
569                 }
570 
571                 if (!vp_dev->per_vq_vectors || msix_vec == VIRTIO_MSI_NO_VECTOR)
572                         continue;
573 
574                 /* allocate per-vq irq if available and necessary */
575                 snprintf(vp_dev->msix_names[msix_vec],
576                          sizeof *vp_dev->msix_names,
577                          "%s-%s",
578                          dev_name(&vp_dev->vdev.dev), names[i]);
579                 err = request_irq(vp_dev->msix_entries[msix_vec].vector,
580                                   vring_interrupt, 0,
581                                   vp_dev->msix_names[msix_vec],
582                                   vqs[i]);
583                 if (err) {
584                         vp_del_vq(vqs[i]);
585                         goto error_find;
586                 }
587         }
588         return 0;
589 
590 error_find:
591         vp_del_vqs(vdev);
592 
593 error_request:
594         return err;
595 }
596 
597 /* the config->find_vqs() implementation */
598 static int vp_find_vqs(struct virtio_device *vdev, unsigned nvqs,
599                        struct virtqueue *vqs[],
600                        vq_callback_t *callbacks[],
601                        const char *names[])
602 {
603         int err;
604 
605         /* Try MSI-X with one vector per queue. */
606         err = vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names, true, true);
607         if (!err)
608                 return 0;
609         /* Fallback: MSI-X with one vector for config, one shared for queues. */
610         err = vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names,
611                                  true, false);
612         if (!err)
613                 return 0;
614         /* Finally fall back to regular interrupts. */
615         return vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names,
616                                   false, false);
617 }
618 
619 static const char *vp_bus_name(struct virtio_device *vdev)
620 {
621         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
622 
623         return pci_name(vp_dev->pci_dev);
624 }
625 
626 /* Setup the affinity for a virtqueue:
627  * - force the affinity for per vq vector
628  * - OR over all affinities for shared MSI
629  * - ignore the affinity request if we're using INTX
630  */
631 static int vp_set_vq_affinity(struct virtqueue *vq, int cpu)
632 {
633         struct virtio_device *vdev = vq->vdev;
634         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
635         struct virtio_pci_vq_info *info = vq->priv;
636         struct cpumask *mask;
637         unsigned int irq;
638 
639         if (!vq->callback)
640                 return -EINVAL;
641 
642         if (vp_dev->msix_enabled) {
643                 mask = vp_dev->msix_affinity_masks[info->msix_vector];
644                 irq = vp_dev->msix_entries[info->msix_vector].vector;
645                 if (cpu == -1)
646                         irq_set_affinity_hint(irq, NULL);
647                 else {
648                         cpumask_set_cpu(cpu, mask);
649                         irq_set_affinity_hint(irq, mask);
650                 }
651         }
652         return 0;
653 }
654 
655 static const struct virtio_config_ops virtio_pci_config_ops = {
656         .get            = vp_get,
657         .set            = vp_set,
658         .get_status     = vp_get_status,
659         .set_status     = vp_set_status,
660         .reset          = vp_reset,
661         .find_vqs       = vp_find_vqs,
662         .del_vqs        = vp_del_vqs,
663         .get_features   = vp_get_features,
664         .finalize_features = vp_finalize_features,
665         .bus_name       = vp_bus_name,
666         .set_vq_affinity = vp_set_vq_affinity,
667 };
668 
669 static void virtio_pci_release_dev(struct device *_d)
670 {
671         /*
672          * No need for a release method as we allocate/free
673          * all devices together with the pci devices.
674          * Provide an empty one to avoid getting a warning from core.
675          */
676 }
677 
678 /* the PCI probing function */
679 static int virtio_pci_probe(struct pci_dev *pci_dev,
680                             const struct pci_device_id *id)
681 {
682         struct virtio_pci_device *vp_dev;
683         int err;
684 
685         /* We only own devices >= 0x1000 and <= 0x103f: leave the rest. */
686         if (pci_dev->device < 0x1000 || pci_dev->device > 0x103f)
687                 return -ENODEV;
688 
689         if (pci_dev->revision != VIRTIO_PCI_ABI_VERSION) {
690                 printk(KERN_ERR "virtio_pci: expected ABI version %d, got %d\n",
691                        VIRTIO_PCI_ABI_VERSION, pci_dev->revision);
692                 return -ENODEV;
693         }
694 
695         /* allocate our structure and fill it out */
696         vp_dev = kzalloc(sizeof(struct virtio_pci_device), GFP_KERNEL);
697         if (vp_dev == NULL)
698                 return -ENOMEM;
699 
700         vp_dev->vdev.dev.parent = &pci_dev->dev;
701         vp_dev->vdev.dev.release = virtio_pci_release_dev;
702         vp_dev->vdev.config = &virtio_pci_config_ops;
703         vp_dev->pci_dev = pci_dev;
704         INIT_LIST_HEAD(&vp_dev->virtqueues);
705         spin_lock_init(&vp_dev->lock);
706 
707         /* Disable MSI/MSIX to bring device to a known good state. */
708         pci_msi_off(pci_dev);
709 
710         /* enable the device */
711         err = pci_enable_device(pci_dev);
712         if (err)
713                 goto out;
714 
715         err = pci_request_regions(pci_dev, "virtio-pci");
716         if (err)
717                 goto out_enable_device;
718 
719         vp_dev->ioaddr = pci_iomap(pci_dev, 0, 0);
720         if (vp_dev->ioaddr == NULL) {
721                 err = -ENOMEM;
722                 goto out_req_regions;
723         }
724 
725         pci_set_drvdata(pci_dev, vp_dev);
726         pci_set_master(pci_dev);
727 
728         /* we use the subsystem vendor/device id as the virtio vendor/device
729          * id.  this allows us to use the same PCI vendor/device id for all
730          * virtio devices and to identify the particular virtio driver by
731          * the subsystem ids */
732         vp_dev->vdev.id.vendor = pci_dev->subsystem_vendor;
733         vp_dev->vdev.id.device = pci_dev->subsystem_device;
734 
735         /* finally register the virtio device */
736         err = register_virtio_device(&vp_dev->vdev);
737         if (err)
738                 goto out_set_drvdata;
739 
740         return 0;
741 
742 out_set_drvdata:
743         pci_iounmap(pci_dev, vp_dev->ioaddr);
744 out_req_regions:
745         pci_release_regions(pci_dev);
746 out_enable_device:
747         pci_disable_device(pci_dev);
748 out:
749         kfree(vp_dev);
750         return err;
751 }
752 
753 static void virtio_pci_remove(struct pci_dev *pci_dev)
754 {
755         struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
756 
757         unregister_virtio_device(&vp_dev->vdev);
758 
759         vp_del_vqs(&vp_dev->vdev);
760         pci_iounmap(pci_dev, vp_dev->ioaddr);
761         pci_release_regions(pci_dev);
762         pci_disable_device(pci_dev);
763         kfree(vp_dev);
764 }
765 
766 #ifdef CONFIG_PM_SLEEP
767 static int virtio_pci_freeze(struct device *dev)
768 {
769         struct pci_dev *pci_dev = to_pci_dev(dev);
770         struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
771         struct virtio_driver *drv;
772         int ret;
773 
774         drv = container_of(vp_dev->vdev.dev.driver,
775                            struct virtio_driver, driver);
776 
777         ret = 0;
778         vp_dev->saved_status = vp_get_status(&vp_dev->vdev);
779         if (drv && drv->freeze)
780                 ret = drv->freeze(&vp_dev->vdev);
781 
782         if (!ret)
783                 pci_disable_device(pci_dev);
784         return ret;
785 }
786 
787 static int virtio_pci_restore(struct device *dev)
788 {
789         struct pci_dev *pci_dev = to_pci_dev(dev);
790         struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
791         struct virtio_driver *drv;
792         int ret;
793 
794         drv = container_of(vp_dev->vdev.dev.driver,
795                            struct virtio_driver, driver);
796 
797         ret = pci_enable_device(pci_dev);
798         if (ret)
799                 return ret;
800 
801         pci_set_master(pci_dev);
802         vp_finalize_features(&vp_dev->vdev);
803 
804         if (drv && drv->restore)
805                 ret = drv->restore(&vp_dev->vdev);
806 
807         /* Finally, tell the device we're all set */
808         if (!ret)
809                 vp_set_status(&vp_dev->vdev, vp_dev->saved_status);
810 
811         return ret;
812 }
813 
814 static const struct dev_pm_ops virtio_pci_pm_ops = {
815         SET_SYSTEM_SLEEP_PM_OPS(virtio_pci_freeze, virtio_pci_restore)
816 };
817 #endif
818 
819 static struct pci_driver virtio_pci_driver = {
820         .name           = "virtio-pci",
821         .id_table       = virtio_pci_id_table,
822         .probe          = virtio_pci_probe,
823         .remove         = virtio_pci_remove,
824 #ifdef CONFIG_PM_SLEEP
825         .driver.pm      = &virtio_pci_pm_ops,
826 #endif
827 };
828 
829 module_pci_driver(virtio_pci_driver);
830 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us