Statistics
| Branch: | Revision:

root / hw / virtio-pci.c @ a1bc20df

History | View | Annotate | Download (32.1 kB)

1
/*
2
 * Virtio PCI Bindings
3
 *
4
 * Copyright IBM, Corp. 2007
5
 * Copyright (c) 2009 CodeSourcery
6
 *
7
 * Authors:
8
 *  Anthony Liguori   <aliguori@us.ibm.com>
9
 *  Paul Brook        <paul@codesourcery.com>
10
 *
11
 * This work is licensed under the terms of the GNU GPL, version 2.  See
12
 * the COPYING file in the top-level directory.
13
 *
14
 * Contributions after 2012-01-13 are licensed under the terms of the
15
 * GNU GPL, version 2 or (at your option) any later version.
16
 */
17

    
18
#include <inttypes.h>
19

    
20
#include "virtio.h"
21
#include "virtio-blk.h"
22
#include "virtio-net.h"
23
#include "virtio-serial.h"
24
#include "virtio-scsi.h"
25
#include "pci.h"
26
#include "qemu-error.h"
27
#include "msi.h"
28
#include "msix.h"
29
#include "net.h"
30
#include "loader.h"
31
#include "kvm.h"
32
#include "blockdev.h"
33
#include "virtio-pci.h"
34
#include "range.h"
35

    
36
/* from Linux's linux/virtio_pci.h */
37

    
38
/* A 32-bit r/o bitmask of the features supported by the host */
39
#define VIRTIO_PCI_HOST_FEATURES        0
40

    
41
/* A 32-bit r/w bitmask of features activated by the guest */
42
#define VIRTIO_PCI_GUEST_FEATURES       4
43

    
44
/* A 32-bit r/w PFN for the currently selected queue */
45
#define VIRTIO_PCI_QUEUE_PFN            8
46

    
47
/* A 16-bit r/o queue size for the currently selected queue */
48
#define VIRTIO_PCI_QUEUE_NUM            12
49

    
50
/* A 16-bit r/w queue selector */
51
#define VIRTIO_PCI_QUEUE_SEL            14
52

    
53
/* A 16-bit r/w queue notifier */
54
#define VIRTIO_PCI_QUEUE_NOTIFY         16
55

    
56
/* An 8-bit device status register.  */
57
#define VIRTIO_PCI_STATUS               18
58

    
59
/* An 8-bit r/o interrupt status register.  Reading the value will return the
60
 * current contents of the ISR and will also clear it.  This is effectively
61
 * a read-and-acknowledge. */
62
#define VIRTIO_PCI_ISR                  19
63

    
64
/* MSI-X registers: only enabled if MSI-X is enabled. */
65
/* A 16-bit vector for configuration changes. */
66
#define VIRTIO_MSI_CONFIG_VECTOR        20
67
/* A 16-bit vector for selected queue notifications. */
68
#define VIRTIO_MSI_QUEUE_VECTOR         22
69

    
70
/* Config space size */
71
#define VIRTIO_PCI_CONFIG_NOMSI         20
72
#define VIRTIO_PCI_CONFIG_MSI           24
73
#define VIRTIO_PCI_REGION_SIZE(dev)     (msix_present(dev) ? \
74
                                         VIRTIO_PCI_CONFIG_MSI : \
75
                                         VIRTIO_PCI_CONFIG_NOMSI)
76

    
77
/* The remaining space is defined by each driver as the per-driver
78
 * configuration space */
79
#define VIRTIO_PCI_CONFIG(dev)          (msix_enabled(dev) ? \
80
                                         VIRTIO_PCI_CONFIG_MSI : \
81
                                         VIRTIO_PCI_CONFIG_NOMSI)
82

    
83
/* How many bits to shift physical queue address written to QUEUE_PFN.
84
 * 12 is historical, and due to x86 page size. */
85
#define VIRTIO_PCI_QUEUE_ADDR_SHIFT    12
86

    
87
/* Flags track per-device state like workarounds for quirks in older guests. */
88
#define VIRTIO_PCI_FLAG_BUS_MASTER_BUG  (1 << 0)
89

    
90
/* QEMU doesn't strictly need write barriers since everything runs in
91
 * lock-step.  We'll leave the calls to wmb() in though to make it obvious for
92
 * KVM or if kqemu gets SMP support.
93
 */
94
#define wmb() do { } while (0)
95

    
96
/* HACK for virtio to determine if it's running a big endian guest */
97
bool virtio_is_big_endian(void);
98

    
99
/* virtio device */
100

    
101
static void virtio_pci_notify(void *opaque, uint16_t vector)
102
{
103
    VirtIOPCIProxy *proxy = opaque;
104
    if (msix_enabled(&proxy->pci_dev))
105
        msix_notify(&proxy->pci_dev, vector);
106
    else
107
        qemu_set_irq(proxy->pci_dev.irq[0], proxy->vdev->isr & 1);
108
}
109

    
110
static void virtio_pci_save_config(void * opaque, QEMUFile *f)
111
{
112
    VirtIOPCIProxy *proxy = opaque;
113
    pci_device_save(&proxy->pci_dev, f);
114
    msix_save(&proxy->pci_dev, f);
115
    if (msix_present(&proxy->pci_dev))
116
        qemu_put_be16(f, proxy->vdev->config_vector);
117
}
118

    
119
static void virtio_pci_save_queue(void * opaque, int n, QEMUFile *f)
120
{
121
    VirtIOPCIProxy *proxy = opaque;
122
    if (msix_present(&proxy->pci_dev))
123
        qemu_put_be16(f, virtio_queue_vector(proxy->vdev, n));
124
}
125

    
126
static int virtio_pci_load_config(void * opaque, QEMUFile *f)
127
{
128
    VirtIOPCIProxy *proxy = opaque;
129
    int ret;
130
    ret = pci_device_load(&proxy->pci_dev, f);
131
    if (ret) {
132
        return ret;
133
    }
134
    msix_unuse_all_vectors(&proxy->pci_dev);
135
    msix_load(&proxy->pci_dev, f);
136
    if (msix_present(&proxy->pci_dev)) {
137
        qemu_get_be16s(f, &proxy->vdev->config_vector);
138
    } else {
139
        proxy->vdev->config_vector = VIRTIO_NO_VECTOR;
140
    }
141
    if (proxy->vdev->config_vector != VIRTIO_NO_VECTOR) {
142
        return msix_vector_use(&proxy->pci_dev, proxy->vdev->config_vector);
143
    }
144
    return 0;
145
}
146

    
147
static int virtio_pci_load_queue(void * opaque, int n, QEMUFile *f)
148
{
149
    VirtIOPCIProxy *proxy = opaque;
150
    uint16_t vector;
151
    if (msix_present(&proxy->pci_dev)) {
152
        qemu_get_be16s(f, &vector);
153
    } else {
154
        vector = VIRTIO_NO_VECTOR;
155
    }
156
    virtio_queue_set_vector(proxy->vdev, n, vector);
157
    if (vector != VIRTIO_NO_VECTOR) {
158
        return msix_vector_use(&proxy->pci_dev, vector);
159
    }
160
    return 0;
161
}
162

    
163
static int virtio_pci_set_host_notifier_internal(VirtIOPCIProxy *proxy,
164
                                                 int n, bool assign, bool set_handler)
165
{
166
    VirtQueue *vq = virtio_get_queue(proxy->vdev, n);
167
    EventNotifier *notifier = virtio_queue_get_host_notifier(vq);
168
    int r = 0;
169

    
170
    if (assign) {
171
        r = event_notifier_init(notifier, 1);
172
        if (r < 0) {
173
            error_report("%s: unable to init event notifier: %d",
174
                         __func__, r);
175
            return r;
176
        }
177
        virtio_queue_set_host_notifier_fd_handler(vq, true, set_handler);
178
        memory_region_add_eventfd(&proxy->bar, VIRTIO_PCI_QUEUE_NOTIFY, 2,
179
                                  true, n, notifier);
180
    } else {
181
        memory_region_del_eventfd(&proxy->bar, VIRTIO_PCI_QUEUE_NOTIFY, 2,
182
                                  true, n, notifier);
183
        virtio_queue_set_host_notifier_fd_handler(vq, false, false);
184
        event_notifier_cleanup(notifier);
185
    }
186
    return r;
187
}
188

    
189
static void virtio_pci_start_ioeventfd(VirtIOPCIProxy *proxy)
190
{
191
    int n, r;
192

    
193
    if (!(proxy->flags & VIRTIO_PCI_FLAG_USE_IOEVENTFD) ||
194
        proxy->ioeventfd_disabled ||
195
        proxy->ioeventfd_started) {
196
        return;
197
    }
198

    
199
    for (n = 0; n < VIRTIO_PCI_QUEUE_MAX; n++) {
200
        if (!virtio_queue_get_num(proxy->vdev, n)) {
201
            continue;
202
        }
203

    
204
        r = virtio_pci_set_host_notifier_internal(proxy, n, true, true);
205
        if (r < 0) {
206
            goto assign_error;
207
        }
208
    }
209
    proxy->ioeventfd_started = true;
210
    return;
211

    
212
assign_error:
213
    while (--n >= 0) {
214
        if (!virtio_queue_get_num(proxy->vdev, n)) {
215
            continue;
216
        }
217

    
218
        r = virtio_pci_set_host_notifier_internal(proxy, n, false, false);
219
        assert(r >= 0);
220
    }
221
    proxy->ioeventfd_started = false;
222
    error_report("%s: failed. Fallback to a userspace (slower).", __func__);
223
}
224

    
225
static void virtio_pci_stop_ioeventfd(VirtIOPCIProxy *proxy)
226
{
227
    int r;
228
    int n;
229

    
230
    if (!proxy->ioeventfd_started) {
231
        return;
232
    }
233

    
234
    for (n = 0; n < VIRTIO_PCI_QUEUE_MAX; n++) {
235
        if (!virtio_queue_get_num(proxy->vdev, n)) {
236
            continue;
237
        }
238

    
239
        r = virtio_pci_set_host_notifier_internal(proxy, n, false, false);
240
        assert(r >= 0);
241
    }
242
    proxy->ioeventfd_started = false;
243
}
244

    
245
void virtio_pci_reset(DeviceState *d)
246
{
247
    VirtIOPCIProxy *proxy = container_of(d, VirtIOPCIProxy, pci_dev.qdev);
248
    virtio_pci_stop_ioeventfd(proxy);
249
    virtio_reset(proxy->vdev);
250
    msix_unuse_all_vectors(&proxy->pci_dev);
251
    proxy->flags &= ~VIRTIO_PCI_FLAG_BUS_MASTER_BUG;
252
}
253

    
254
static void virtio_ioport_write(void *opaque, uint32_t addr, uint32_t val)
255
{
256
    VirtIOPCIProxy *proxy = opaque;
257
    VirtIODevice *vdev = proxy->vdev;
258
    hwaddr pa;
259

    
260
    switch (addr) {
261
    case VIRTIO_PCI_GUEST_FEATURES:
262
        /* Guest does not negotiate properly?  We have to assume nothing. */
263
        if (val & (1 << VIRTIO_F_BAD_FEATURE)) {
264
            val = vdev->bad_features ? vdev->bad_features(vdev) : 0;
265
        }
266
        virtio_set_features(vdev, val);
267
        break;
268
    case VIRTIO_PCI_QUEUE_PFN:
269
        pa = (hwaddr)val << VIRTIO_PCI_QUEUE_ADDR_SHIFT;
270
        if (pa == 0) {
271
            virtio_pci_stop_ioeventfd(proxy);
272
            virtio_reset(proxy->vdev);
273
            msix_unuse_all_vectors(&proxy->pci_dev);
274
        }
275
        else
276
            virtio_queue_set_addr(vdev, vdev->queue_sel, pa);
277
        break;
278
    case VIRTIO_PCI_QUEUE_SEL:
279
        if (val < VIRTIO_PCI_QUEUE_MAX)
280
            vdev->queue_sel = val;
281
        break;
282
    case VIRTIO_PCI_QUEUE_NOTIFY:
283
        if (val < VIRTIO_PCI_QUEUE_MAX) {
284
            virtio_queue_notify(vdev, val);
285
        }
286
        break;
287
    case VIRTIO_PCI_STATUS:
288
        if (!(val & VIRTIO_CONFIG_S_DRIVER_OK)) {
289
            virtio_pci_stop_ioeventfd(proxy);
290
        }
291

    
292
        virtio_set_status(vdev, val & 0xFF);
293

    
294
        if (val & VIRTIO_CONFIG_S_DRIVER_OK) {
295
            virtio_pci_start_ioeventfd(proxy);
296
        }
297

    
298
        if (vdev->status == 0) {
299
            virtio_reset(proxy->vdev);
300
            msix_unuse_all_vectors(&proxy->pci_dev);
301
        }
302

    
303
        /* Linux before 2.6.34 sets the device as OK without enabling
304
           the PCI device bus master bit. In this case we need to disable
305
           some safety checks. */
306
        if ((val & VIRTIO_CONFIG_S_DRIVER_OK) &&
307
            !(proxy->pci_dev.config[PCI_COMMAND] & PCI_COMMAND_MASTER)) {
308
            proxy->flags |= VIRTIO_PCI_FLAG_BUS_MASTER_BUG;
309
        }
310
        break;
311
    case VIRTIO_MSI_CONFIG_VECTOR:
312
        msix_vector_unuse(&proxy->pci_dev, vdev->config_vector);
313
        /* Make it possible for guest to discover an error took place. */
314
        if (msix_vector_use(&proxy->pci_dev, val) < 0)
315
            val = VIRTIO_NO_VECTOR;
316
        vdev->config_vector = val;
317
        break;
318
    case VIRTIO_MSI_QUEUE_VECTOR:
319
        msix_vector_unuse(&proxy->pci_dev,
320
                          virtio_queue_vector(vdev, vdev->queue_sel));
321
        /* Make it possible for guest to discover an error took place. */
322
        if (msix_vector_use(&proxy->pci_dev, val) < 0)
323
            val = VIRTIO_NO_VECTOR;
324
        virtio_queue_set_vector(vdev, vdev->queue_sel, val);
325
        break;
326
    default:
327
        error_report("%s: unexpected address 0x%x value 0x%x",
328
                     __func__, addr, val);
329
        break;
330
    }
331
}
332

    
333
static uint32_t virtio_ioport_read(VirtIOPCIProxy *proxy, uint32_t addr)
334
{
335
    VirtIODevice *vdev = proxy->vdev;
336
    uint32_t ret = 0xFFFFFFFF;
337

    
338
    switch (addr) {
339
    case VIRTIO_PCI_HOST_FEATURES:
340
        ret = proxy->host_features;
341
        break;
342
    case VIRTIO_PCI_GUEST_FEATURES:
343
        ret = vdev->guest_features;
344
        break;
345
    case VIRTIO_PCI_QUEUE_PFN:
346
        ret = virtio_queue_get_addr(vdev, vdev->queue_sel)
347
              >> VIRTIO_PCI_QUEUE_ADDR_SHIFT;
348
        break;
349
    case VIRTIO_PCI_QUEUE_NUM:
350
        ret = virtio_queue_get_num(vdev, vdev->queue_sel);
351
        break;
352
    case VIRTIO_PCI_QUEUE_SEL:
353
        ret = vdev->queue_sel;
354
        break;
355
    case VIRTIO_PCI_STATUS:
356
        ret = vdev->status;
357
        break;
358
    case VIRTIO_PCI_ISR:
359
        /* reading from the ISR also clears it. */
360
        ret = vdev->isr;
361
        vdev->isr = 0;
362
        qemu_set_irq(proxy->pci_dev.irq[0], 0);
363
        break;
364
    case VIRTIO_MSI_CONFIG_VECTOR:
365
        ret = vdev->config_vector;
366
        break;
367
    case VIRTIO_MSI_QUEUE_VECTOR:
368
        ret = virtio_queue_vector(vdev, vdev->queue_sel);
369
        break;
370
    default:
371
        break;
372
    }
373

    
374
    return ret;
375
}
376

    
377
static uint64_t virtio_pci_config_read(void *opaque, hwaddr addr,
378
                                       unsigned size)
379
{
380
    VirtIOPCIProxy *proxy = opaque;
381
    uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
382
    uint64_t val = 0;
383
    if (addr < config) {
384
        return virtio_ioport_read(proxy, addr);
385
    }
386
    addr -= config;
387

    
388
    switch (size) {
389
    case 1:
390
        val = virtio_config_readb(proxy->vdev, addr);
391
        break;
392
    case 2:
393
        val = virtio_config_readw(proxy->vdev, addr);
394
        if (virtio_is_big_endian()) {
395
            val = bswap16(val);
396
        }
397
        break;
398
    case 4:
399
        val = virtio_config_readl(proxy->vdev, addr);
400
        if (virtio_is_big_endian()) {
401
            val = bswap32(val);
402
        }
403
        break;
404
    }
405
    return val;
406
}
407

    
408
static void virtio_pci_config_write(void *opaque, hwaddr addr,
409
                                    uint64_t val, unsigned size)
410
{
411
    VirtIOPCIProxy *proxy = opaque;
412
    uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
413
    if (addr < config) {
414
        virtio_ioport_write(proxy, addr, val);
415
        return;
416
    }
417
    addr -= config;
418
    /*
419
     * Virtio-PCI is odd. Ioports are LE but config space is target native
420
     * endian.
421
     */
422
    switch (size) {
423
    case 1:
424
        virtio_config_writeb(proxy->vdev, addr, val);
425
        break;
426
    case 2:
427
        if (virtio_is_big_endian()) {
428
            val = bswap16(val);
429
        }
430
        virtio_config_writew(proxy->vdev, addr, val);
431
        break;
432
    case 4:
433
        if (virtio_is_big_endian()) {
434
            val = bswap32(val);
435
        }
436
        virtio_config_writel(proxy->vdev, addr, val);
437
        break;
438
    }
439
}
440

    
441
static const MemoryRegionOps virtio_pci_config_ops = {
442
    .read = virtio_pci_config_read,
443
    .write = virtio_pci_config_write,
444
    .impl = {
445
        .min_access_size = 1,
446
        .max_access_size = 4,
447
    },
448
    .endianness = DEVICE_LITTLE_ENDIAN,
449
};
450

    
451
static void virtio_write_config(PCIDevice *pci_dev, uint32_t address,
452
                                uint32_t val, int len)
453
{
454
    VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
455

    
456
    pci_default_write_config(pci_dev, address, val, len);
457

    
458
    if (range_covers_byte(address, len, PCI_COMMAND) &&
459
        !(pci_dev->config[PCI_COMMAND] & PCI_COMMAND_MASTER) &&
460
        !(proxy->flags & VIRTIO_PCI_FLAG_BUS_MASTER_BUG)) {
461
        virtio_pci_stop_ioeventfd(proxy);
462
        virtio_set_status(proxy->vdev,
463
                          proxy->vdev->status & ~VIRTIO_CONFIG_S_DRIVER_OK);
464
    }
465
}
466

    
467
static unsigned virtio_pci_get_features(void *opaque)
468
{
469
    VirtIOPCIProxy *proxy = opaque;
470
    return proxy->host_features;
471
}
472

    
473
static int kvm_virtio_pci_vq_vector_use(VirtIOPCIProxy *proxy,
474
                                        unsigned int queue_no,
475
                                        unsigned int vector,
476
                                        MSIMessage msg)
477
{
478
    VirtQueue *vq = virtio_get_queue(proxy->vdev, queue_no);
479
    EventNotifier *n = virtio_queue_get_guest_notifier(vq);
480
    VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
481
    int ret;
482

    
483
    if (irqfd->users == 0) {
484
        ret = kvm_irqchip_add_msi_route(kvm_state, msg);
485
        if (ret < 0) {
486
            return ret;
487
        }
488
        irqfd->virq = ret;
489
    }
490
    irqfd->users++;
491

    
492
    ret = kvm_irqchip_add_irqfd_notifier(kvm_state, n, irqfd->virq);
493
    if (ret < 0) {
494
        if (--irqfd->users == 0) {
495
            kvm_irqchip_release_virq(kvm_state, irqfd->virq);
496
        }
497
        return ret;
498
    }
499

    
500
    virtio_queue_set_guest_notifier_fd_handler(vq, true, true);
501
    return 0;
502
}
503

    
504
static void kvm_virtio_pci_vq_vector_release(VirtIOPCIProxy *proxy,
505
                                             unsigned int queue_no,
506
                                             unsigned int vector)
507
{
508
    VirtQueue *vq = virtio_get_queue(proxy->vdev, queue_no);
509
    EventNotifier *n = virtio_queue_get_guest_notifier(vq);
510
    VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
511
    int ret;
512

    
513
    ret = kvm_irqchip_remove_irqfd_notifier(kvm_state, n, irqfd->virq);
514
    assert(ret == 0);
515

    
516
    if (--irqfd->users == 0) {
517
        kvm_irqchip_release_virq(kvm_state, irqfd->virq);
518
    }
519

    
520
    virtio_queue_set_guest_notifier_fd_handler(vq, true, false);
521
}
522

    
523
static int kvm_virtio_pci_vector_use(PCIDevice *dev, unsigned vector,
524
                                     MSIMessage msg)
525
{
526
    VirtIOPCIProxy *proxy = container_of(dev, VirtIOPCIProxy, pci_dev);
527
    VirtIODevice *vdev = proxy->vdev;
528
    int ret, queue_no;
529

    
530
    for (queue_no = 0; queue_no < VIRTIO_PCI_QUEUE_MAX; queue_no++) {
531
        if (!virtio_queue_get_num(vdev, queue_no)) {
532
            break;
533
        }
534
        if (virtio_queue_vector(vdev, queue_no) != vector) {
535
            continue;
536
        }
537
        ret = kvm_virtio_pci_vq_vector_use(proxy, queue_no, vector, msg);
538
        if (ret < 0) {
539
            goto undo;
540
        }
541
    }
542
    return 0;
543

    
544
undo:
545
    while (--queue_no >= 0) {
546
        if (virtio_queue_vector(vdev, queue_no) != vector) {
547
            continue;
548
        }
549
        kvm_virtio_pci_vq_vector_release(proxy, queue_no, vector);
550
    }
551
    return ret;
552
}
553

    
554
static void kvm_virtio_pci_vector_release(PCIDevice *dev, unsigned vector)
555
{
556
    VirtIOPCIProxy *proxy = container_of(dev, VirtIOPCIProxy, pci_dev);
557
    VirtIODevice *vdev = proxy->vdev;
558
    int queue_no;
559

    
560
    for (queue_no = 0; queue_no < VIRTIO_PCI_QUEUE_MAX; queue_no++) {
561
        if (!virtio_queue_get_num(vdev, queue_no)) {
562
            break;
563
        }
564
        if (virtio_queue_vector(vdev, queue_no) != vector) {
565
            continue;
566
        }
567
        kvm_virtio_pci_vq_vector_release(proxy, queue_no, vector);
568
    }
569
}
570

    
571
static int virtio_pci_set_guest_notifier(void *opaque, int n, bool assign)
572
{
573
    VirtIOPCIProxy *proxy = opaque;
574
    VirtQueue *vq = virtio_get_queue(proxy->vdev, n);
575
    EventNotifier *notifier = virtio_queue_get_guest_notifier(vq);
576

    
577
    if (assign) {
578
        int r = event_notifier_init(notifier, 0);
579
        if (r < 0) {
580
            return r;
581
        }
582
        virtio_queue_set_guest_notifier_fd_handler(vq, true, false);
583
    } else {
584
        virtio_queue_set_guest_notifier_fd_handler(vq, false, false);
585
        event_notifier_cleanup(notifier);
586
    }
587

    
588
    return 0;
589
}
590

    
591
static bool virtio_pci_query_guest_notifiers(void *opaque)
592
{
593
    VirtIOPCIProxy *proxy = opaque;
594
    return msix_enabled(&proxy->pci_dev);
595
}
596

    
597
static int virtio_pci_set_guest_notifiers(void *opaque, bool assign)
598
{
599
    VirtIOPCIProxy *proxy = opaque;
600
    VirtIODevice *vdev = proxy->vdev;
601
    int r, n;
602

    
603
    /* Must unset vector notifier while guest notifier is still assigned */
604
    if (kvm_msi_via_irqfd_enabled() && !assign) {
605
        msix_unset_vector_notifiers(&proxy->pci_dev);
606
        g_free(proxy->vector_irqfd);
607
        proxy->vector_irqfd = NULL;
608
    }
609

    
610
    for (n = 0; n < VIRTIO_PCI_QUEUE_MAX; n++) {
611
        if (!virtio_queue_get_num(vdev, n)) {
612
            break;
613
        }
614

    
615
        r = virtio_pci_set_guest_notifier(opaque, n, assign);
616
        if (r < 0) {
617
            goto assign_error;
618
        }
619
    }
620

    
621
    /* Must set vector notifier after guest notifier has been assigned */
622
    if (kvm_msi_via_irqfd_enabled() && assign) {
623
        proxy->vector_irqfd =
624
            g_malloc0(sizeof(*proxy->vector_irqfd) *
625
                      msix_nr_vectors_allocated(&proxy->pci_dev));
626
        r = msix_set_vector_notifiers(&proxy->pci_dev,
627
                                      kvm_virtio_pci_vector_use,
628
                                      kvm_virtio_pci_vector_release);
629
        if (r < 0) {
630
            goto assign_error;
631
        }
632
    }
633

    
634
    return 0;
635

    
636
assign_error:
637
    /* We get here on assignment failure. Recover by undoing for VQs 0 .. n. */
638
    assert(assign);
639
    while (--n >= 0) {
640
        virtio_pci_set_guest_notifier(opaque, n, !assign);
641
    }
642
    return r;
643
}
644

    
645
static int virtio_pci_set_host_notifier(void *opaque, int n, bool assign)
646
{
647
    VirtIOPCIProxy *proxy = opaque;
648

    
649
    /* Stop using ioeventfd for virtqueue kick if the device starts using host
650
     * notifiers.  This makes it easy to avoid stepping on each others' toes.
651
     */
652
    proxy->ioeventfd_disabled = assign;
653
    if (assign) {
654
        virtio_pci_stop_ioeventfd(proxy);
655
    }
656
    /* We don't need to start here: it's not needed because backend
657
     * currently only stops on status change away from ok,
658
     * reset, vmstop and such. If we do add code to start here,
659
     * need to check vmstate, device state etc. */
660
    return virtio_pci_set_host_notifier_internal(proxy, n, assign, false);
661
}
662

    
663
static void virtio_pci_vmstate_change(void *opaque, bool running)
664
{
665
    VirtIOPCIProxy *proxy = opaque;
666

    
667
    if (running) {
668
        /* Try to find out if the guest has bus master disabled, but is
669
           in ready state. Then we have a buggy guest OS. */
670
        if ((proxy->vdev->status & VIRTIO_CONFIG_S_DRIVER_OK) &&
671
            !(proxy->pci_dev.config[PCI_COMMAND] & PCI_COMMAND_MASTER)) {
672
            proxy->flags |= VIRTIO_PCI_FLAG_BUS_MASTER_BUG;
673
        }
674
        virtio_pci_start_ioeventfd(proxy);
675
    } else {
676
        virtio_pci_stop_ioeventfd(proxy);
677
    }
678
}
679

    
680
static const VirtIOBindings virtio_pci_bindings = {
681
    .notify = virtio_pci_notify,
682
    .save_config = virtio_pci_save_config,
683
    .load_config = virtio_pci_load_config,
684
    .save_queue = virtio_pci_save_queue,
685
    .load_queue = virtio_pci_load_queue,
686
    .get_features = virtio_pci_get_features,
687
    .query_guest_notifiers = virtio_pci_query_guest_notifiers,
688
    .set_host_notifier = virtio_pci_set_host_notifier,
689
    .set_guest_notifiers = virtio_pci_set_guest_notifiers,
690
    .vmstate_change = virtio_pci_vmstate_change,
691
};
692

    
693
void virtio_init_pci(VirtIOPCIProxy *proxy, VirtIODevice *vdev)
694
{
695
    uint8_t *config;
696
    uint32_t size;
697

    
698
    proxy->vdev = vdev;
699

    
700
    config = proxy->pci_dev.config;
701

    
702
    if (proxy->class_code) {
703
        pci_config_set_class(config, proxy->class_code);
704
    }
705
    pci_set_word(config + PCI_SUBSYSTEM_VENDOR_ID,
706
                 pci_get_word(config + PCI_VENDOR_ID));
707
    pci_set_word(config + PCI_SUBSYSTEM_ID, vdev->device_id);
708
    config[PCI_INTERRUPT_PIN] = 1;
709

    
710
    if (vdev->nvectors &&
711
        msix_init_exclusive_bar(&proxy->pci_dev, vdev->nvectors, 1)) {
712
        vdev->nvectors = 0;
713
    }
714

    
715
    proxy->pci_dev.config_write = virtio_write_config;
716

    
717
    size = VIRTIO_PCI_REGION_SIZE(&proxy->pci_dev) + vdev->config_len;
718
    if (size & (size-1))
719
        size = 1 << qemu_fls(size);
720

    
721
    memory_region_init_io(&proxy->bar, &virtio_pci_config_ops, proxy,
722
                          "virtio-pci", size);
723
    pci_register_bar(&proxy->pci_dev, 0, PCI_BASE_ADDRESS_SPACE_IO,
724
                     &proxy->bar);
725

    
726
    if (!kvm_has_many_ioeventfds()) {
727
        proxy->flags &= ~VIRTIO_PCI_FLAG_USE_IOEVENTFD;
728
    }
729

    
730
    virtio_bind_device(vdev, &virtio_pci_bindings, proxy);
731
    proxy->host_features |= 0x1 << VIRTIO_F_NOTIFY_ON_EMPTY;
732
    proxy->host_features |= 0x1 << VIRTIO_F_BAD_FEATURE;
733
    proxy->host_features = vdev->get_features(vdev, proxy->host_features);
734
}
735

    
736
static int virtio_blk_init_pci(PCIDevice *pci_dev)
737
{
738
    VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
739
    VirtIODevice *vdev;
740

    
741
    if (proxy->class_code != PCI_CLASS_STORAGE_SCSI &&
742
        proxy->class_code != PCI_CLASS_STORAGE_OTHER)
743
        proxy->class_code = PCI_CLASS_STORAGE_SCSI;
744

    
745
    vdev = virtio_blk_init(&pci_dev->qdev, &proxy->blk);
746
    if (!vdev) {
747
        return -1;
748
    }
749
    vdev->nvectors = proxy->nvectors;
750
    virtio_init_pci(proxy, vdev);
751
    /* make the actual value visible */
752
    proxy->nvectors = vdev->nvectors;
753
    return 0;
754
}
755

    
756
static void virtio_exit_pci(PCIDevice *pci_dev)
757
{
758
    VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
759

    
760
    memory_region_destroy(&proxy->bar);
761
    msix_uninit_exclusive_bar(pci_dev);
762
}
763

    
764
static void virtio_blk_exit_pci(PCIDevice *pci_dev)
765
{
766
    VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
767

    
768
    virtio_pci_stop_ioeventfd(proxy);
769
    virtio_blk_exit(proxy->vdev);
770
    virtio_exit_pci(pci_dev);
771
}
772

    
773
static int virtio_serial_init_pci(PCIDevice *pci_dev)
774
{
775
    VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
776
    VirtIODevice *vdev;
777

    
778
    if (proxy->class_code != PCI_CLASS_COMMUNICATION_OTHER &&
779
        proxy->class_code != PCI_CLASS_DISPLAY_OTHER && /* qemu 0.10 */
780
        proxy->class_code != PCI_CLASS_OTHERS)          /* qemu-kvm  */
781
        proxy->class_code = PCI_CLASS_COMMUNICATION_OTHER;
782

    
783
    vdev = virtio_serial_init(&pci_dev->qdev, &proxy->serial);
784
    if (!vdev) {
785
        return -1;
786
    }
787
    vdev->nvectors = proxy->nvectors == DEV_NVECTORS_UNSPECIFIED
788
                                        ? proxy->serial.max_virtserial_ports + 1
789
                                        : proxy->nvectors;
790
    virtio_init_pci(proxy, vdev);
791
    proxy->nvectors = vdev->nvectors;
792
    return 0;
793
}
794

    
795
static void virtio_serial_exit_pci(PCIDevice *pci_dev)
796
{
797
    VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
798

    
799
    virtio_pci_stop_ioeventfd(proxy);
800
    virtio_serial_exit(proxy->vdev);
801
    virtio_exit_pci(pci_dev);
802
}
803

    
804
static int virtio_net_init_pci(PCIDevice *pci_dev)
805
{
806
    VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
807
    VirtIODevice *vdev;
808

    
809
    vdev = virtio_net_init(&pci_dev->qdev, &proxy->nic, &proxy->net);
810

    
811
    vdev->nvectors = proxy->nvectors;
812
    virtio_init_pci(proxy, vdev);
813

    
814
    /* make the actual value visible */
815
    proxy->nvectors = vdev->nvectors;
816
    return 0;
817
}
818

    
819
static void virtio_net_exit_pci(PCIDevice *pci_dev)
820
{
821
    VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
822

    
823
    virtio_pci_stop_ioeventfd(proxy);
824
    virtio_net_exit(proxy->vdev);
825
    virtio_exit_pci(pci_dev);
826
}
827

    
828
static int virtio_balloon_init_pci(PCIDevice *pci_dev)
829
{
830
    VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
831
    VirtIODevice *vdev;
832

    
833
    if (proxy->class_code != PCI_CLASS_OTHERS &&
834
        proxy->class_code != PCI_CLASS_MEMORY_RAM) { /* qemu < 1.1 */
835
        proxy->class_code = PCI_CLASS_OTHERS;
836
    }
837

    
838
    vdev = virtio_balloon_init(&pci_dev->qdev);
839
    if (!vdev) {
840
        return -1;
841
    }
842
    virtio_init_pci(proxy, vdev);
843
    return 0;
844
}
845

    
846
static void virtio_balloon_exit_pci(PCIDevice *pci_dev)
847
{
848
    VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
849

    
850
    virtio_pci_stop_ioeventfd(proxy);
851
    virtio_balloon_exit(proxy->vdev);
852
    virtio_exit_pci(pci_dev);
853
}
854

    
855
static Property virtio_blk_properties[] = {
856
    DEFINE_PROP_HEX32("class", VirtIOPCIProxy, class_code, 0),
857
    DEFINE_BLOCK_PROPERTIES(VirtIOPCIProxy, blk.conf),
858
    DEFINE_BLOCK_CHS_PROPERTIES(VirtIOPCIProxy, blk.conf),
859
    DEFINE_PROP_STRING("serial", VirtIOPCIProxy, blk.serial),
860
#ifdef __linux__
861
    DEFINE_PROP_BIT("scsi", VirtIOPCIProxy, blk.scsi, 0, true),
862
#endif
863
    DEFINE_PROP_BIT("config-wce", VirtIOPCIProxy, blk.config_wce, 0, true),
864
    DEFINE_PROP_BIT("ioeventfd", VirtIOPCIProxy, flags, VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT, true),
865
    DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors, 2),
866
    DEFINE_VIRTIO_BLK_FEATURES(VirtIOPCIProxy, host_features),
867
    DEFINE_PROP_END_OF_LIST(),
868
};
869

    
870
static void virtio_blk_class_init(ObjectClass *klass, void *data)
871
{
872
    DeviceClass *dc = DEVICE_CLASS(klass);
873
    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
874

    
875
    k->init = virtio_blk_init_pci;
876
    k->exit = virtio_blk_exit_pci;
877
    k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
878
    k->device_id = PCI_DEVICE_ID_VIRTIO_BLOCK;
879
    k->revision = VIRTIO_PCI_ABI_VERSION;
880
    k->class_id = PCI_CLASS_STORAGE_SCSI;
881
    dc->reset = virtio_pci_reset;
882
    dc->props = virtio_blk_properties;
883
}
884

    
885
static TypeInfo virtio_blk_info = {
886
    .name          = "virtio-blk-pci",
887
    .parent        = TYPE_PCI_DEVICE,
888
    .instance_size = sizeof(VirtIOPCIProxy),
889
    .class_init    = virtio_blk_class_init,
890
};
891

    
892
static Property virtio_net_properties[] = {
893
    DEFINE_PROP_BIT("ioeventfd", VirtIOPCIProxy, flags, VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT, false),
894
    DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors, 3),
895
    DEFINE_VIRTIO_NET_FEATURES(VirtIOPCIProxy, host_features),
896
    DEFINE_NIC_PROPERTIES(VirtIOPCIProxy, nic),
897
    DEFINE_PROP_UINT32("x-txtimer", VirtIOPCIProxy, net.txtimer, TX_TIMER_INTERVAL),
898
    DEFINE_PROP_INT32("x-txburst", VirtIOPCIProxy, net.txburst, TX_BURST),
899
    DEFINE_PROP_STRING("tx", VirtIOPCIProxy, net.tx),
900
    DEFINE_PROP_END_OF_LIST(),
901
};
902

    
903
static void virtio_net_class_init(ObjectClass *klass, void *data)
904
{
905
    DeviceClass *dc = DEVICE_CLASS(klass);
906
    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
907

    
908
    k->init = virtio_net_init_pci;
909
    k->exit = virtio_net_exit_pci;
910
    k->romfile = "pxe-virtio.rom";
911
    k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
912
    k->device_id = PCI_DEVICE_ID_VIRTIO_NET;
913
    k->revision = VIRTIO_PCI_ABI_VERSION;
914
    k->class_id = PCI_CLASS_NETWORK_ETHERNET;
915
    dc->reset = virtio_pci_reset;
916
    dc->props = virtio_net_properties;
917
}
918

    
919
static TypeInfo virtio_net_info = {
920
    .name          = "virtio-net-pci",
921
    .parent        = TYPE_PCI_DEVICE,
922
    .instance_size = sizeof(VirtIOPCIProxy),
923
    .class_init    = virtio_net_class_init,
924
};
925

    
926
static Property virtio_serial_properties[] = {
927
    DEFINE_PROP_BIT("ioeventfd", VirtIOPCIProxy, flags, VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT, true),
928
    DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors, DEV_NVECTORS_UNSPECIFIED),
929
    DEFINE_PROP_HEX32("class", VirtIOPCIProxy, class_code, 0),
930
    DEFINE_VIRTIO_COMMON_FEATURES(VirtIOPCIProxy, host_features),
931
    DEFINE_PROP_UINT32("max_ports", VirtIOPCIProxy, serial.max_virtserial_ports, 31),
932
    DEFINE_PROP_END_OF_LIST(),
933
};
934

    
935
static void virtio_serial_class_init(ObjectClass *klass, void *data)
936
{
937
    DeviceClass *dc = DEVICE_CLASS(klass);
938
    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
939

    
940
    k->init = virtio_serial_init_pci;
941
    k->exit = virtio_serial_exit_pci;
942
    k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
943
    k->device_id = PCI_DEVICE_ID_VIRTIO_CONSOLE;
944
    k->revision = VIRTIO_PCI_ABI_VERSION;
945
    k->class_id = PCI_CLASS_COMMUNICATION_OTHER;
946
    dc->reset = virtio_pci_reset;
947
    dc->props = virtio_serial_properties;
948
}
949

    
950
static TypeInfo virtio_serial_info = {
951
    .name          = "virtio-serial-pci",
952
    .parent        = TYPE_PCI_DEVICE,
953
    .instance_size = sizeof(VirtIOPCIProxy),
954
    .class_init    = virtio_serial_class_init,
955
};
956

    
957
static Property virtio_balloon_properties[] = {
958
    DEFINE_VIRTIO_COMMON_FEATURES(VirtIOPCIProxy, host_features),
959
    DEFINE_PROP_HEX32("class", VirtIOPCIProxy, class_code, 0),
960
    DEFINE_PROP_END_OF_LIST(),
961
};
962

    
963
static void virtio_balloon_class_init(ObjectClass *klass, void *data)
964
{
965
    DeviceClass *dc = DEVICE_CLASS(klass);
966
    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
967

    
968
    k->init = virtio_balloon_init_pci;
969
    k->exit = virtio_balloon_exit_pci;
970
    k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
971
    k->device_id = PCI_DEVICE_ID_VIRTIO_BALLOON;
972
    k->revision = VIRTIO_PCI_ABI_VERSION;
973
    k->class_id = PCI_CLASS_OTHERS;
974
    dc->reset = virtio_pci_reset;
975
    dc->props = virtio_balloon_properties;
976
}
977

    
978
static TypeInfo virtio_balloon_info = {
979
    .name          = "virtio-balloon-pci",
980
    .parent        = TYPE_PCI_DEVICE,
981
    .instance_size = sizeof(VirtIOPCIProxy),
982
    .class_init    = virtio_balloon_class_init,
983
};
984

    
985
static int virtio_scsi_init_pci(PCIDevice *pci_dev)
986
{
987
    VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
988
    VirtIODevice *vdev;
989

    
990
    vdev = virtio_scsi_init(&pci_dev->qdev, &proxy->scsi);
991
    if (!vdev) {
992
        return -EINVAL;
993
    }
994

    
995
    vdev->nvectors = proxy->nvectors == DEV_NVECTORS_UNSPECIFIED
996
                                        ? proxy->scsi.num_queues + 3
997
                                        : proxy->nvectors;
998
    virtio_init_pci(proxy, vdev);
999

    
1000
    /* make the actual value visible */
1001
    proxy->nvectors = vdev->nvectors;
1002
    return 0;
1003
}
1004

    
1005
static void virtio_scsi_exit_pci(PCIDevice *pci_dev)
1006
{
1007
    VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
1008

    
1009
    virtio_scsi_exit(proxy->vdev);
1010
    virtio_exit_pci(pci_dev);
1011
}
1012

    
1013
static Property virtio_scsi_properties[] = {
1014
    DEFINE_PROP_BIT("ioeventfd", VirtIOPCIProxy, flags, VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT, true),
1015
    DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors, DEV_NVECTORS_UNSPECIFIED),
1016
    DEFINE_VIRTIO_SCSI_PROPERTIES(VirtIOPCIProxy, host_features, scsi),
1017
    DEFINE_PROP_END_OF_LIST(),
1018
};
1019

    
1020
static void virtio_scsi_class_init(ObjectClass *klass, void *data)
1021
{
1022
    DeviceClass *dc = DEVICE_CLASS(klass);
1023
    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
1024

    
1025
    k->init = virtio_scsi_init_pci;
1026
    k->exit = virtio_scsi_exit_pci;
1027
    k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
1028
    k->device_id = PCI_DEVICE_ID_VIRTIO_SCSI;
1029
    k->revision = 0x00;
1030
    k->class_id = PCI_CLASS_STORAGE_SCSI;
1031
    dc->reset = virtio_pci_reset;
1032
    dc->props = virtio_scsi_properties;
1033
}
1034

    
1035
static TypeInfo virtio_scsi_info = {
1036
    .name          = "virtio-scsi-pci",
1037
    .parent        = TYPE_PCI_DEVICE,
1038
    .instance_size = sizeof(VirtIOPCIProxy),
1039
    .class_init    = virtio_scsi_class_init,
1040
};
1041

    
1042
static void virtio_pci_register_types(void)
1043
{
1044
    type_register_static(&virtio_blk_info);
1045
    type_register_static(&virtio_net_info);
1046
    type_register_static(&virtio_serial_info);
1047
    type_register_static(&virtio_balloon_info);
1048
    type_register_static(&virtio_scsi_info);
1049
}
1050

    
1051
type_init(virtio_pci_register_types)