Version:  2.0.40 2.2.26 2.4.37 3.2 3.3 3.4 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16 3.17 3.18

Linux/drivers/virtio/virtio_pci.c

  1 /*
  2  * Virtio PCI driver
  3  *
  4  * This module allows virtio devices to be used over a virtual PCI device.
  5  * This can be used with QEMU based VMMs like KVM or Xen.
  6  *
  7  * Copyright IBM Corp. 2007
  8  *
  9  * Authors:
 10  *  Anthony Liguori  <aliguori@us.ibm.com>
 11  *
 12  * This work is licensed under the terms of the GNU GPL, version 2 or later.
 13  * See the COPYING file in the top-level directory.
 14  *
 15  */
 16 
 17 #include <linux/module.h>
 18 #include <linux/list.h>
 19 #include <linux/pci.h>
 20 #include <linux/slab.h>
 21 #include <linux/interrupt.h>
 22 #include <linux/virtio.h>
 23 #include <linux/virtio_config.h>
 24 #include <linux/virtio_ring.h>
 25 #include <linux/virtio_pci.h>
 26 #include <linux/highmem.h>
 27 #include <linux/spinlock.h>
 28 
 29 MODULE_AUTHOR("Anthony Liguori <aliguori@us.ibm.com>");
 30 MODULE_DESCRIPTION("virtio-pci");
 31 MODULE_LICENSE("GPL");
 32 MODULE_VERSION("1");
 33 
 34 /* Our device structure */
 35 struct virtio_pci_device
 36 {
 37         struct virtio_device vdev;
 38         struct pci_dev *pci_dev;
 39 
 40         /* the IO mapping for the PCI config space */
 41         void __iomem *ioaddr;
 42 
 43         /* a list of queues so we can dispatch IRQs */
 44         spinlock_t lock;
 45         struct list_head virtqueues;
 46 
 47         /* MSI-X support */
 48         int msix_enabled;
 49         int intx_enabled;
 50         struct msix_entry *msix_entries;
 51         cpumask_var_t *msix_affinity_masks;
 52         /* Name strings for interrupts. This size should be enough,
 53          * and I'm too lazy to allocate each name separately. */
 54         char (*msix_names)[256];
 55         /* Number of available vectors */
 56         unsigned msix_vectors;
 57         /* Vectors allocated, excluding per-vq vectors if any */
 58         unsigned msix_used_vectors;
 59 
 60         /* Whether we have vector per vq */
 61         bool per_vq_vectors;
 62 };
 63 
 64 /* Constants for MSI-X */
 65 /* Use first vector for configuration changes, second and the rest for
 66  * virtqueues Thus, we need at least 2 vectors for MSI. */
 67 enum {
 68         VP_MSIX_CONFIG_VECTOR = 0,
 69         VP_MSIX_VQ_VECTOR = 1,
 70 };
 71 
 72 struct virtio_pci_vq_info
 73 {
 74         /* the actual virtqueue */
 75         struct virtqueue *vq;
 76 
 77         /* the number of entries in the queue */
 78         int num;
 79 
 80         /* the virtual address of the ring queue */
 81         void *queue;
 82 
 83         /* the list node for the virtqueues list */
 84         struct list_head node;
 85 
 86         /* MSI-X vector (or none) */
 87         unsigned msix_vector;
 88 };
 89 
 90 /* Qumranet donated their vendor ID for devices 0x1000 thru 0x10FF. */
 91 static const struct pci_device_id virtio_pci_id_table[] = {
 92         { PCI_DEVICE(0x1af4, PCI_ANY_ID) },
 93         { 0 }
 94 };
 95 
 96 MODULE_DEVICE_TABLE(pci, virtio_pci_id_table);
 97 
 98 /* Convert a generic virtio device to our structure */
 99 static struct virtio_pci_device *to_vp_device(struct virtio_device *vdev)
100 {
101         return container_of(vdev, struct virtio_pci_device, vdev);
102 }
103 
104 /* virtio config->get_features() implementation */
105 static u32 vp_get_features(struct virtio_device *vdev)
106 {
107         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
108 
109         /* When someone needs more than 32 feature bits, we'll need to
110          * steal a bit to indicate that the rest are somewhere else. */
111         return ioread32(vp_dev->ioaddr + VIRTIO_PCI_HOST_FEATURES);
112 }
113 
114 /* virtio config->finalize_features() implementation */
115 static void vp_finalize_features(struct virtio_device *vdev)
116 {
117         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
118 
119         /* Give virtio_ring a chance to accept features. */
120         vring_transport_features(vdev);
121 
122         /* We only support 32 feature bits. */
123         BUILD_BUG_ON(ARRAY_SIZE(vdev->features) != 1);
124         iowrite32(vdev->features[0], vp_dev->ioaddr+VIRTIO_PCI_GUEST_FEATURES);
125 }
126 
127 /* virtio config->get() implementation */
128 static void vp_get(struct virtio_device *vdev, unsigned offset,
129                    void *buf, unsigned len)
130 {
131         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
132         void __iomem *ioaddr = vp_dev->ioaddr +
133                                 VIRTIO_PCI_CONFIG(vp_dev) + offset;
134         u8 *ptr = buf;
135         int i;
136 
137         for (i = 0; i < len; i++)
138                 ptr[i] = ioread8(ioaddr + i);
139 }
140 
141 /* the config->set() implementation.  it's symmetric to the config->get()
142  * implementation */
143 static void vp_set(struct virtio_device *vdev, unsigned offset,
144                    const void *buf, unsigned len)
145 {
146         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
147         void __iomem *ioaddr = vp_dev->ioaddr +
148                                 VIRTIO_PCI_CONFIG(vp_dev) + offset;
149         const u8 *ptr = buf;
150         int i;
151 
152         for (i = 0; i < len; i++)
153                 iowrite8(ptr[i], ioaddr + i);
154 }
155 
156 /* config->{get,set}_status() implementations */
157 static u8 vp_get_status(struct virtio_device *vdev)
158 {
159         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
160         return ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
161 }
162 
163 static void vp_set_status(struct virtio_device *vdev, u8 status)
164 {
165         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
166         /* We should never be setting status to 0. */
167         BUG_ON(status == 0);
168         iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
169 }
170 
171 /* wait for pending irq handlers */
172 static void vp_synchronize_vectors(struct virtio_device *vdev)
173 {
174         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
175         int i;
176 
177         if (vp_dev->intx_enabled)
178                 synchronize_irq(vp_dev->pci_dev->irq);
179 
180         for (i = 0; i < vp_dev->msix_vectors; ++i)
181                 synchronize_irq(vp_dev->msix_entries[i].vector);
182 }
183 
184 static void vp_reset(struct virtio_device *vdev)
185 {
186         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
187         /* 0 status means a reset. */
188         iowrite8(0, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
189         /* Flush out the status write, and flush in device writes,
190          * including MSi-X interrupts, if any. */
191         ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
192         /* Flush pending VQ/configuration callbacks. */
193         vp_synchronize_vectors(vdev);
194 }
195 
196 /* the notify function used when creating a virt queue */
197 static bool vp_notify(struct virtqueue *vq)
198 {
199         struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
200 
201         /* we write the queue's selector into the notification register to
202          * signal the other end */
203         iowrite16(vq->index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_NOTIFY);
204         return true;
205 }
206 
207 /* Handle a configuration change: Tell driver if it wants to know. */
208 static irqreturn_t vp_config_changed(int irq, void *opaque)
209 {
210         struct virtio_pci_device *vp_dev = opaque;
211 
212         virtio_config_changed(&vp_dev->vdev);
213         return IRQ_HANDLED;
214 }
215 
216 /* Notify all virtqueues on an interrupt. */
217 static irqreturn_t vp_vring_interrupt(int irq, void *opaque)
218 {
219         struct virtio_pci_device *vp_dev = opaque;
220         struct virtio_pci_vq_info *info;
221         irqreturn_t ret = IRQ_NONE;
222         unsigned long flags;
223 
224         spin_lock_irqsave(&vp_dev->lock, flags);
225         list_for_each_entry(info, &vp_dev->virtqueues, node) {
226                 if (vring_interrupt(irq, info->vq) == IRQ_HANDLED)
227                         ret = IRQ_HANDLED;
228         }
229         spin_unlock_irqrestore(&vp_dev->lock, flags);
230 
231         return ret;
232 }
233 
234 /* A small wrapper to also acknowledge the interrupt when it's handled.
235  * I really need an EIO hook for the vring so I can ack the interrupt once we
236  * know that we'll be handling the IRQ but before we invoke the callback since
237  * the callback may notify the host which results in the host attempting to
238  * raise an interrupt that we would then mask once we acknowledged the
239  * interrupt. */
240 static irqreturn_t vp_interrupt(int irq, void *opaque)
241 {
242         struct virtio_pci_device *vp_dev = opaque;
243         u8 isr;
244 
245         /* reading the ISR has the effect of also clearing it so it's very
246          * important to save off the value. */
247         isr = ioread8(vp_dev->ioaddr + VIRTIO_PCI_ISR);
248 
249         /* It's definitely not us if the ISR was not high */
250         if (!isr)
251                 return IRQ_NONE;
252 
253         /* Configuration change?  Tell driver if it wants to know. */
254         if (isr & VIRTIO_PCI_ISR_CONFIG)
255                 vp_config_changed(irq, opaque);
256 
257         return vp_vring_interrupt(irq, opaque);
258 }
259 
260 static void vp_free_vectors(struct virtio_device *vdev)
261 {
262         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
263         int i;
264 
265         if (vp_dev->intx_enabled) {
266                 free_irq(vp_dev->pci_dev->irq, vp_dev);
267                 vp_dev->intx_enabled = 0;
268         }
269 
270         for (i = 0; i < vp_dev->msix_used_vectors; ++i)
271                 free_irq(vp_dev->msix_entries[i].vector, vp_dev);
272 
273         for (i = 0; i < vp_dev->msix_vectors; i++)
274                 if (vp_dev->msix_affinity_masks[i])
275                         free_cpumask_var(vp_dev->msix_affinity_masks[i]);
276 
277         if (vp_dev->msix_enabled) {
278                 /* Disable the vector used for configuration */
279                 iowrite16(VIRTIO_MSI_NO_VECTOR,
280                           vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR);
281                 /* Flush the write out to device */
282                 ioread16(vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR);
283 
284                 pci_disable_msix(vp_dev->pci_dev);
285                 vp_dev->msix_enabled = 0;
286         }
287 
288         vp_dev->msix_vectors = 0;
289         vp_dev->msix_used_vectors = 0;
290         kfree(vp_dev->msix_names);
291         vp_dev->msix_names = NULL;
292         kfree(vp_dev->msix_entries);
293         vp_dev->msix_entries = NULL;
294         kfree(vp_dev->msix_affinity_masks);
295         vp_dev->msix_affinity_masks = NULL;
296 }
297 
298 static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors,
299                                    bool per_vq_vectors)
300 {
301         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
302         const char *name = dev_name(&vp_dev->vdev.dev);
303         unsigned i, v;
304         int err = -ENOMEM;
305 
306         vp_dev->msix_vectors = nvectors;
307 
308         vp_dev->msix_entries = kmalloc(nvectors * sizeof *vp_dev->msix_entries,
309                                        GFP_KERNEL);
310         if (!vp_dev->msix_entries)
311                 goto error;
312         vp_dev->msix_names = kmalloc(nvectors * sizeof *vp_dev->msix_names,
313                                      GFP_KERNEL);
314         if (!vp_dev->msix_names)
315                 goto error;
316         vp_dev->msix_affinity_masks
317                 = kzalloc(nvectors * sizeof *vp_dev->msix_affinity_masks,
318                           GFP_KERNEL);
319         if (!vp_dev->msix_affinity_masks)
320                 goto error;
321         for (i = 0; i < nvectors; ++i)
322                 if (!alloc_cpumask_var(&vp_dev->msix_affinity_masks[i],
323                                         GFP_KERNEL))
324                         goto error;
325 
326         for (i = 0; i < nvectors; ++i)
327                 vp_dev->msix_entries[i].entry = i;
328 
329         err = pci_enable_msix_exact(vp_dev->pci_dev,
330                                     vp_dev->msix_entries, nvectors);
331         if (err)
332                 goto error;
333         vp_dev->msix_enabled = 1;
334 
335         /* Set the vector used for configuration */
336         v = vp_dev->msix_used_vectors;
337         snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
338                  "%s-config", name);
339         err = request_irq(vp_dev->msix_entries[v].vector,
340                           vp_config_changed, 0, vp_dev->msix_names[v],
341                           vp_dev);
342         if (err)
343                 goto error;
344         ++vp_dev->msix_used_vectors;
345 
346         iowrite16(v, vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR);
347         /* Verify we had enough resources to assign the vector */
348         v = ioread16(vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR);
349         if (v == VIRTIO_MSI_NO_VECTOR) {
350                 err = -EBUSY;
351                 goto error;
352         }
353 
354         if (!per_vq_vectors) {
355                 /* Shared vector for all VQs */
356                 v = vp_dev->msix_used_vectors;
357                 snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
358                          "%s-virtqueues", name);
359                 err = request_irq(vp_dev->msix_entries[v].vector,
360                                   vp_vring_interrupt, 0, vp_dev->msix_names[v],
361                                   vp_dev);
362                 if (err)
363                         goto error;
364                 ++vp_dev->msix_used_vectors;
365         }
366         return 0;
367 error:
368         vp_free_vectors(vdev);
369         return err;
370 }
371 
372 static int vp_request_intx(struct virtio_device *vdev)
373 {
374         int err;
375         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
376 
377         err = request_irq(vp_dev->pci_dev->irq, vp_interrupt,
378                           IRQF_SHARED, dev_name(&vdev->dev), vp_dev);
379         if (!err)
380                 vp_dev->intx_enabled = 1;
381         return err;
382 }
383 
384 static struct virtqueue *setup_vq(struct virtio_device *vdev, unsigned index,
385                                   void (*callback)(struct virtqueue *vq),
386                                   const char *name,
387                                   u16 msix_vec)
388 {
389         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
390         struct virtio_pci_vq_info *info;
391         struct virtqueue *vq;
392         unsigned long flags, size;
393         u16 num;
394         int err;
395 
396         /* Select the queue we're interested in */
397         iowrite16(index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_SEL);
398 
399         /* Check if queue is either not available or already active. */
400         num = ioread16(vp_dev->ioaddr + VIRTIO_PCI_QUEUE_NUM);
401         if (!num || ioread32(vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN))
402                 return ERR_PTR(-ENOENT);
403 
404         /* allocate and fill out our structure the represents an active
405          * queue */
406         info = kmalloc(sizeof(struct virtio_pci_vq_info), GFP_KERNEL);
407         if (!info)
408                 return ERR_PTR(-ENOMEM);
409 
410         info->num = num;
411         info->msix_vector = msix_vec;
412 
413         size = PAGE_ALIGN(vring_size(num, VIRTIO_PCI_VRING_ALIGN));
414         info->queue = alloc_pages_exact(size, GFP_KERNEL|__GFP_ZERO);
415         if (info->queue == NULL) {
416                 err = -ENOMEM;
417                 goto out_info;
418         }
419 
420         /* activate the queue */
421         iowrite32(virt_to_phys(info->queue) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT,
422                   vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
423 
424         /* create the vring */
425         vq = vring_new_virtqueue(index, info->num, VIRTIO_PCI_VRING_ALIGN, vdev,
426                                  true, info->queue, vp_notify, callback, name);
427         if (!vq) {
428                 err = -ENOMEM;
429                 goto out_activate_queue;
430         }
431 
432         vq->priv = info;
433         info->vq = vq;
434 
435         if (msix_vec != VIRTIO_MSI_NO_VECTOR) {
436                 iowrite16(msix_vec, vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR);
437                 msix_vec = ioread16(vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR);
438                 if (msix_vec == VIRTIO_MSI_NO_VECTOR) {
439                         err = -EBUSY;
440                         goto out_assign;
441                 }
442         }
443 
444         if (callback) {
445                 spin_lock_irqsave(&vp_dev->lock, flags);
446                 list_add(&info->node, &vp_dev->virtqueues);
447                 spin_unlock_irqrestore(&vp_dev->lock, flags);
448         } else {
449                 INIT_LIST_HEAD(&info->node);
450         }
451 
452         return vq;
453 
454 out_assign:
455         vring_del_virtqueue(vq);
456 out_activate_queue:
457         iowrite32(0, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
458         free_pages_exact(info->queue, size);
459 out_info:
460         kfree(info);
461         return ERR_PTR(err);
462 }
463 
464 static void vp_del_vq(struct virtqueue *vq)
465 {
466         struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
467         struct virtio_pci_vq_info *info = vq->priv;
468         unsigned long flags, size;
469 
470         spin_lock_irqsave(&vp_dev->lock, flags);
471         list_del(&info->node);
472         spin_unlock_irqrestore(&vp_dev->lock, flags);
473 
474         iowrite16(vq->index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_SEL);
475 
476         if (vp_dev->msix_enabled) {
477                 iowrite16(VIRTIO_MSI_NO_VECTOR,
478                           vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR);
479                 /* Flush the write out to device */
480                 ioread8(vp_dev->ioaddr + VIRTIO_PCI_ISR);
481         }
482 
483         vring_del_virtqueue(vq);
484 
485         /* Select and deactivate the queue */
486         iowrite32(0, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
487 
488         size = PAGE_ALIGN(vring_size(info->num, VIRTIO_PCI_VRING_ALIGN));
489         free_pages_exact(info->queue, size);
490         kfree(info);
491 }
492 
493 /* the config->del_vqs() implementation */
494 static void vp_del_vqs(struct virtio_device *vdev)
495 {
496         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
497         struct virtqueue *vq, *n;
498         struct virtio_pci_vq_info *info;
499 
500         list_for_each_entry_safe(vq, n, &vdev->vqs, list) {
501                 info = vq->priv;
502                 if (vp_dev->per_vq_vectors &&
503                         info->msix_vector != VIRTIO_MSI_NO_VECTOR)
504                         free_irq(vp_dev->msix_entries[info->msix_vector].vector,
505                                  vq);
506                 vp_del_vq(vq);
507         }
508         vp_dev->per_vq_vectors = false;
509 
510         vp_free_vectors(vdev);
511 }
512 
513 static int vp_try_to_find_vqs(struct virtio_device *vdev, unsigned nvqs,
514                               struct virtqueue *vqs[],
515                               vq_callback_t *callbacks[],
516                               const char *names[],
517                               bool use_msix,
518                               bool per_vq_vectors)
519 {
520         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
521         u16 msix_vec;
522         int i, err, nvectors, allocated_vectors;
523 
524         if (!use_msix) {
525                 /* Old style: one normal interrupt for change and all vqs. */
526                 err = vp_request_intx(vdev);
527                 if (err)
528                         goto error_request;
529         } else {
530                 if (per_vq_vectors) {
531                         /* Best option: one for change interrupt, one per vq. */
532                         nvectors = 1;
533                         for (i = 0; i < nvqs; ++i)
534                                 if (callbacks[i])
535                                         ++nvectors;
536                 } else {
537                         /* Second best: one for change, shared for all vqs. */
538                         nvectors = 2;
539                 }
540 
541                 err = vp_request_msix_vectors(vdev, nvectors, per_vq_vectors);
542                 if (err)
543                         goto error_request;
544         }
545 
546         vp_dev->per_vq_vectors = per_vq_vectors;
547         allocated_vectors = vp_dev->msix_used_vectors;
548         for (i = 0; i < nvqs; ++i) {
549                 if (!names[i]) {
550                         vqs[i] = NULL;
551                         continue;
552                 } else if (!callbacks[i] || !vp_dev->msix_enabled)
553                         msix_vec = VIRTIO_MSI_NO_VECTOR;
554                 else if (vp_dev->per_vq_vectors)
555                         msix_vec = allocated_vectors++;
556                 else
557                         msix_vec = VP_MSIX_VQ_VECTOR;
558                 vqs[i] = setup_vq(vdev, i, callbacks[i], names[i], msix_vec);
559                 if (IS_ERR(vqs[i])) {
560                         err = PTR_ERR(vqs[i]);
561                         goto error_find;
562                 }
563 
564                 if (!vp_dev->per_vq_vectors || msix_vec == VIRTIO_MSI_NO_VECTOR)
565                         continue;
566 
567                 /* allocate per-vq irq if available and necessary */
568                 snprintf(vp_dev->msix_names[msix_vec],
569                          sizeof *vp_dev->msix_names,
570                          "%s-%s",
571                          dev_name(&vp_dev->vdev.dev), names[i]);
572                 err = request_irq(vp_dev->msix_entries[msix_vec].vector,
573                                   vring_interrupt, 0,
574                                   vp_dev->msix_names[msix_vec],
575                                   vqs[i]);
576                 if (err) {
577                         vp_del_vq(vqs[i]);
578                         goto error_find;
579                 }
580         }
581         return 0;
582 
583 error_find:
584         vp_del_vqs(vdev);
585 
586 error_request:
587         return err;
588 }
589 
590 /* the config->find_vqs() implementation */
591 static int vp_find_vqs(struct virtio_device *vdev, unsigned nvqs,
592                        struct virtqueue *vqs[],
593                        vq_callback_t *callbacks[],
594                        const char *names[])
595 {
596         int err;
597 
598         /* Try MSI-X with one vector per queue. */
599         err = vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names, true, true);
600         if (!err)
601                 return 0;
602         /* Fallback: MSI-X with one vector for config, one shared for queues. */
603         err = vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names,
604                                  true, false);
605         if (!err)
606                 return 0;
607         /* Finally fall back to regular interrupts. */
608         return vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names,
609                                   false, false);
610 }
611 
612 static const char *vp_bus_name(struct virtio_device *vdev)
613 {
614         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
615 
616         return pci_name(vp_dev->pci_dev);
617 }
618 
619 /* Setup the affinity for a virtqueue:
620  * - force the affinity for per vq vector
621  * - OR over all affinities for shared MSI
622  * - ignore the affinity request if we're using INTX
623  */
624 static int vp_set_vq_affinity(struct virtqueue *vq, int cpu)
625 {
626         struct virtio_device *vdev = vq->vdev;
627         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
628         struct virtio_pci_vq_info *info = vq->priv;
629         struct cpumask *mask;
630         unsigned int irq;
631 
632         if (!vq->callback)
633                 return -EINVAL;
634 
635         if (vp_dev->msix_enabled) {
636                 mask = vp_dev->msix_affinity_masks[info->msix_vector];
637                 irq = vp_dev->msix_entries[info->msix_vector].vector;
638                 if (cpu == -1)
639                         irq_set_affinity_hint(irq, NULL);
640                 else {
641                         cpumask_set_cpu(cpu, mask);
642                         irq_set_affinity_hint(irq, mask);
643                 }
644         }
645         return 0;
646 }
647 
648 static const struct virtio_config_ops virtio_pci_config_ops = {
649         .get            = vp_get,
650         .set            = vp_set,
651         .get_status     = vp_get_status,
652         .set_status     = vp_set_status,
653         .reset          = vp_reset,
654         .find_vqs       = vp_find_vqs,
655         .del_vqs        = vp_del_vqs,
656         .get_features   = vp_get_features,
657         .finalize_features = vp_finalize_features,
658         .bus_name       = vp_bus_name,
659         .set_vq_affinity = vp_set_vq_affinity,
660 };
661 
662 static void virtio_pci_release_dev(struct device *_d)
663 {
664         /*
665          * No need for a release method as we allocate/free
666          * all devices together with the pci devices.
667          * Provide an empty one to avoid getting a warning from core.
668          */
669 }
670 
671 /* the PCI probing function */
672 static int virtio_pci_probe(struct pci_dev *pci_dev,
673                             const struct pci_device_id *id)
674 {
675         struct virtio_pci_device *vp_dev;
676         int err;
677 
678         /* We only own devices >= 0x1000 and <= 0x103f: leave the rest. */
679         if (pci_dev->device < 0x1000 || pci_dev->device > 0x103f)
680                 return -ENODEV;
681 
682         if (pci_dev->revision != VIRTIO_PCI_ABI_VERSION) {
683                 printk(KERN_ERR "virtio_pci: expected ABI version %d, got %d\n",
684                        VIRTIO_PCI_ABI_VERSION, pci_dev->revision);
685                 return -ENODEV;
686         }
687 
688         /* allocate our structure and fill it out */
689         vp_dev = kzalloc(sizeof(struct virtio_pci_device), GFP_KERNEL);
690         if (vp_dev == NULL)
691                 return -ENOMEM;
692 
693         vp_dev->vdev.dev.parent = &pci_dev->dev;
694         vp_dev->vdev.dev.release = virtio_pci_release_dev;
695         vp_dev->vdev.config = &virtio_pci_config_ops;
696         vp_dev->pci_dev = pci_dev;
697         INIT_LIST_HEAD(&vp_dev->virtqueues);
698         spin_lock_init(&vp_dev->lock);
699 
700         /* Disable MSI/MSIX to bring device to a known good state. */
701         pci_msi_off(pci_dev);
702 
703         /* enable the device */
704         err = pci_enable_device(pci_dev);
705         if (err)
706                 goto out;
707 
708         err = pci_request_regions(pci_dev, "virtio-pci");
709         if (err)
710                 goto out_enable_device;
711 
712         vp_dev->ioaddr = pci_iomap(pci_dev, 0, 0);
713         if (vp_dev->ioaddr == NULL) {
714                 err = -ENOMEM;
715                 goto out_req_regions;
716         }
717 
718         pci_set_drvdata(pci_dev, vp_dev);
719         pci_set_master(pci_dev);
720 
721         /* we use the subsystem vendor/device id as the virtio vendor/device
722          * id.  this allows us to use the same PCI vendor/device id for all
723          * virtio devices and to identify the particular virtio driver by
724          * the subsystem ids */
725         vp_dev->vdev.id.vendor = pci_dev->subsystem_vendor;
726         vp_dev->vdev.id.device = pci_dev->subsystem_device;
727 
728         /* finally register the virtio device */
729         err = register_virtio_device(&vp_dev->vdev);
730         if (err)
731                 goto out_set_drvdata;
732 
733         return 0;
734 
735 out_set_drvdata:
736         pci_iounmap(pci_dev, vp_dev->ioaddr);
737 out_req_regions:
738         pci_release_regions(pci_dev);
739 out_enable_device:
740         pci_disable_device(pci_dev);
741 out:
742         kfree(vp_dev);
743         return err;
744 }
745 
746 static void virtio_pci_remove(struct pci_dev *pci_dev)
747 {
748         struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
749 
750         unregister_virtio_device(&vp_dev->vdev);
751 
752         vp_del_vqs(&vp_dev->vdev);
753         pci_iounmap(pci_dev, vp_dev->ioaddr);
754         pci_release_regions(pci_dev);
755         pci_disable_device(pci_dev);
756         kfree(vp_dev);
757 }
758 
759 #ifdef CONFIG_PM_SLEEP
760 static int virtio_pci_freeze(struct device *dev)
761 {
762         struct pci_dev *pci_dev = to_pci_dev(dev);
763         struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
764         int ret;
765 
766         ret = virtio_device_freeze(&vp_dev->vdev);
767 
768         if (!ret)
769                 pci_disable_device(pci_dev);
770         return ret;
771 }
772 
773 static int virtio_pci_restore(struct device *dev)
774 {
775         struct pci_dev *pci_dev = to_pci_dev(dev);
776         struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
777         int ret;
778 
779         ret = pci_enable_device(pci_dev);
780         if (ret)
781                 return ret;
782 
783         pci_set_master(pci_dev);
784         return virtio_device_restore(&vp_dev->vdev);
785 }
786 
787 static const struct dev_pm_ops virtio_pci_pm_ops = {
788         SET_SYSTEM_SLEEP_PM_OPS(virtio_pci_freeze, virtio_pci_restore)
789 };
790 #endif
791 
792 static struct pci_driver virtio_pci_driver = {
793         .name           = "virtio-pci",
794         .id_table       = virtio_pci_id_table,
795         .probe          = virtio_pci_probe,
796         .remove         = virtio_pci_remove,
797 #ifdef CONFIG_PM_SLEEP
798         .driver.pm      = &virtio_pci_pm_ops,
799 #endif
800 };
801 
802 module_pci_driver(virtio_pci_driver);
803 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us