Statistics
| Branch: | Revision:

root / kvm-all.c @ 93148aa5

History | View | Annotate | Download (42 kB)

1
/*
2
 * QEMU KVM support
3
 *
4
 * Copyright IBM, Corp. 2008
5
 *           Red Hat, Inc. 2008
6
 *
7
 * Authors:
8
 *  Anthony Liguori   <aliguori@us.ibm.com>
9
 *  Glauber Costa     <gcosta@redhat.com>
10
 *
11
 * This work is licensed under the terms of the GNU GPL, version 2 or later.
12
 * See the COPYING file in the top-level directory.
13
 *
14
 */
15

    
16
#include <sys/types.h>
17
#include <sys/ioctl.h>
18
#include <sys/mman.h>
19
#include <stdarg.h>
20

    
21
#include <linux/kvm.h>
22

    
23
#include "qemu-common.h"
24
#include "qemu-barrier.h"
25
#include "sysemu.h"
26
#include "hw/hw.h"
27
#include "gdbstub.h"
28
#include "kvm.h"
29
#include "bswap.h"
30
#include "memory.h"
31
#include "exec-memory.h"
32

    
33
/* This check must be after config-host.h is included */
34
#ifdef CONFIG_EVENTFD
35
#include <sys/eventfd.h>
36
#endif
37

    
38
/* KVM uses PAGE_SIZE in its definition of COALESCED_MMIO_MAX */
39
#define PAGE_SIZE TARGET_PAGE_SIZE
40

    
41
//#define DEBUG_KVM
42

    
43
#ifdef DEBUG_KVM
44
#define DPRINTF(fmt, ...) \
45
    do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
46
#else
47
#define DPRINTF(fmt, ...) \
48
    do { } while (0)
49
#endif
50

    
51
typedef struct KVMSlot
52
{
53
    target_phys_addr_t start_addr;
54
    ram_addr_t memory_size;
55
    void *ram;
56
    int slot;
57
    int flags;
58
} KVMSlot;
59

    
60
typedef struct kvm_dirty_log KVMDirtyLog;
61

    
62
struct KVMState
63
{
64
    KVMSlot slots[32];
65
    int fd;
66
    int vmfd;
67
    int coalesced_mmio;
68
    struct kvm_coalesced_mmio_ring *coalesced_mmio_ring;
69
    bool coalesced_flush_in_progress;
70
    int broken_set_mem_region;
71
    int migration_log;
72
    int vcpu_events;
73
    int robust_singlestep;
74
    int debugregs;
75
#ifdef KVM_CAP_SET_GUEST_DEBUG
76
    struct kvm_sw_breakpoint_head kvm_sw_breakpoints;
77
#endif
78
    int pit_in_kernel;
79
    int xsave, xcrs;
80
    int many_ioeventfds;
81
    int irqchip_inject_ioctl;
82
#ifdef KVM_CAP_IRQ_ROUTING
83
    struct kvm_irq_routing *irq_routes;
84
    int nr_allocated_irq_routes;
85
    uint32_t *used_gsi_bitmap;
86
    unsigned int max_gsi;
87
#endif
88
};
89

    
90
KVMState *kvm_state;
91
bool kvm_kernel_irqchip;
92

    
93
static const KVMCapabilityInfo kvm_required_capabilites[] = {
94
    KVM_CAP_INFO(USER_MEMORY),
95
    KVM_CAP_INFO(DESTROY_MEMORY_REGION_WORKS),
96
    KVM_CAP_LAST_INFO
97
};
98

    
99
static KVMSlot *kvm_alloc_slot(KVMState *s)
100
{
101
    int i;
102

    
103
    for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
104
        if (s->slots[i].memory_size == 0) {
105
            return &s->slots[i];
106
        }
107
    }
108

    
109
    fprintf(stderr, "%s: no free slot available\n", __func__);
110
    abort();
111
}
112

    
113
static KVMSlot *kvm_lookup_matching_slot(KVMState *s,
114
                                         target_phys_addr_t start_addr,
115
                                         target_phys_addr_t end_addr)
116
{
117
    int i;
118

    
119
    for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
120
        KVMSlot *mem = &s->slots[i];
121

    
122
        if (start_addr == mem->start_addr &&
123
            end_addr == mem->start_addr + mem->memory_size) {
124
            return mem;
125
        }
126
    }
127

    
128
    return NULL;
129
}
130

    
131
/*
132
 * Find overlapping slot with lowest start address
133
 */
134
static KVMSlot *kvm_lookup_overlapping_slot(KVMState *s,
135
                                            target_phys_addr_t start_addr,
136
                                            target_phys_addr_t end_addr)
137
{
138
    KVMSlot *found = NULL;
139
    int i;
140

    
141
    for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
142
        KVMSlot *mem = &s->slots[i];
143

    
144
        if (mem->memory_size == 0 ||
145
            (found && found->start_addr < mem->start_addr)) {
146
            continue;
147
        }
148

    
149
        if (end_addr > mem->start_addr &&
150
            start_addr < mem->start_addr + mem->memory_size) {
151
            found = mem;
152
        }
153
    }
154

    
155
    return found;
156
}
157

    
158
int kvm_physical_memory_addr_from_host(KVMState *s, void *ram,
159
                                       target_phys_addr_t *phys_addr)
160
{
161
    int i;
162

    
163
    for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
164
        KVMSlot *mem = &s->slots[i];
165

    
166
        if (ram >= mem->ram && ram < mem->ram + mem->memory_size) {
167
            *phys_addr = mem->start_addr + (ram - mem->ram);
168
            return 1;
169
        }
170
    }
171

    
172
    return 0;
173
}
174

    
175
static int kvm_set_user_memory_region(KVMState *s, KVMSlot *slot)
176
{
177
    struct kvm_userspace_memory_region mem;
178

    
179
    mem.slot = slot->slot;
180
    mem.guest_phys_addr = slot->start_addr;
181
    mem.memory_size = slot->memory_size;
182
    mem.userspace_addr = (unsigned long)slot->ram;
183
    mem.flags = slot->flags;
184
    if (s->migration_log) {
185
        mem.flags |= KVM_MEM_LOG_DIRTY_PAGES;
186
    }
187
    return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
188
}
189

    
190
static void kvm_reset_vcpu(void *opaque)
191
{
192
    CPUState *env = opaque;
193

    
194
    kvm_arch_reset_vcpu(env);
195
}
196

    
197
int kvm_pit_in_kernel(void)
198
{
199
    return kvm_state->pit_in_kernel;
200
}
201

    
202
int kvm_init_vcpu(CPUState *env)
203
{
204
    KVMState *s = kvm_state;
205
    long mmap_size;
206
    int ret;
207

    
208
    DPRINTF("kvm_init_vcpu\n");
209

    
210
    ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, env->cpu_index);
211
    if (ret < 0) {
212
        DPRINTF("kvm_create_vcpu failed\n");
213
        goto err;
214
    }
215

    
216
    env->kvm_fd = ret;
217
    env->kvm_state = s;
218
    env->kvm_vcpu_dirty = 1;
219

    
220
    mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
221
    if (mmap_size < 0) {
222
        ret = mmap_size;
223
        DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n");
224
        goto err;
225
    }
226

    
227
    env->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
228
                        env->kvm_fd, 0);
229
    if (env->kvm_run == MAP_FAILED) {
230
        ret = -errno;
231
        DPRINTF("mmap'ing vcpu state failed\n");
232
        goto err;
233
    }
234

    
235
    if (s->coalesced_mmio && !s->coalesced_mmio_ring) {
236
        s->coalesced_mmio_ring =
237
            (void *)env->kvm_run + s->coalesced_mmio * PAGE_SIZE;
238
    }
239

    
240
    ret = kvm_arch_init_vcpu(env);
241
    if (ret == 0) {
242
        qemu_register_reset(kvm_reset_vcpu, env);
243
        kvm_arch_reset_vcpu(env);
244
    }
245
err:
246
    return ret;
247
}
248

    
249
/*
250
 * dirty pages logging control
251
 */
252

    
253
static int kvm_mem_flags(KVMState *s, bool log_dirty)
254
{
255
    return log_dirty ? KVM_MEM_LOG_DIRTY_PAGES : 0;
256
}
257

    
258
static int kvm_slot_dirty_pages_log_change(KVMSlot *mem, bool log_dirty)
259
{
260
    KVMState *s = kvm_state;
261
    int flags, mask = KVM_MEM_LOG_DIRTY_PAGES;
262
    int old_flags;
263

    
264
    old_flags = mem->flags;
265

    
266
    flags = (mem->flags & ~mask) | kvm_mem_flags(s, log_dirty);
267
    mem->flags = flags;
268

    
269
    /* If nothing changed effectively, no need to issue ioctl */
270
    if (s->migration_log) {
271
        flags |= KVM_MEM_LOG_DIRTY_PAGES;
272
    }
273

    
274
    if (flags == old_flags) {
275
        return 0;
276
    }
277

    
278
    return kvm_set_user_memory_region(s, mem);
279
}
280

    
281
static int kvm_dirty_pages_log_change(target_phys_addr_t phys_addr,
282
                                      ram_addr_t size, bool log_dirty)
283
{
284
    KVMState *s = kvm_state;
285
    KVMSlot *mem = kvm_lookup_matching_slot(s, phys_addr, phys_addr + size);
286

    
287
    if (mem == NULL)  {
288
        fprintf(stderr, "BUG: %s: invalid parameters " TARGET_FMT_plx "-"
289
                TARGET_FMT_plx "\n", __func__, phys_addr,
290
                (target_phys_addr_t)(phys_addr + size - 1));
291
        return -EINVAL;
292
    }
293
    return kvm_slot_dirty_pages_log_change(mem, log_dirty);
294
}
295

    
296
static void kvm_log_start(MemoryListener *listener,
297
                          MemoryRegionSection *section)
298
{
299
    int r;
300

    
301
    r = kvm_dirty_pages_log_change(section->offset_within_address_space,
302
                                   section->size, true);
303
    if (r < 0) {
304
        abort();
305
    }
306
}
307

    
308
static void kvm_log_stop(MemoryListener *listener,
309
                          MemoryRegionSection *section)
310
{
311
    int r;
312

    
313
    r = kvm_dirty_pages_log_change(section->offset_within_address_space,
314
                                   section->size, false);
315
    if (r < 0) {
316
        abort();
317
    }
318
}
319

    
320
static int kvm_set_migration_log(int enable)
321
{
322
    KVMState *s = kvm_state;
323
    KVMSlot *mem;
324
    int i, err;
325

    
326
    s->migration_log = enable;
327

    
328
    for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
329
        mem = &s->slots[i];
330

    
331
        if (!mem->memory_size) {
332
            continue;
333
        }
334
        if (!!(mem->flags & KVM_MEM_LOG_DIRTY_PAGES) == enable) {
335
            continue;
336
        }
337
        err = kvm_set_user_memory_region(s, mem);
338
        if (err) {
339
            return err;
340
        }
341
    }
342
    return 0;
343
}
344

    
345
/* get kvm's dirty pages bitmap and update qemu's */
346
static int kvm_get_dirty_pages_log_range(MemoryRegionSection *section,
347
                                         unsigned long *bitmap)
348
{
349
    unsigned int i, j;
350
    unsigned long page_number, c;
351
    target_phys_addr_t addr, addr1;
352
    unsigned int len = ((section->size / TARGET_PAGE_SIZE) + HOST_LONG_BITS - 1) / HOST_LONG_BITS;
353

    
354
    /*
355
     * bitmap-traveling is faster than memory-traveling (for addr...)
356
     * especially when most of the memory is not dirty.
357
     */
358
    for (i = 0; i < len; i++) {
359
        if (bitmap[i] != 0) {
360
            c = leul_to_cpu(bitmap[i]);
361
            do {
362
                j = ffsl(c) - 1;
363
                c &= ~(1ul << j);
364
                page_number = i * HOST_LONG_BITS + j;
365
                addr1 = page_number * TARGET_PAGE_SIZE;
366
                addr = section->offset_within_region + addr1;
367
                memory_region_set_dirty(section->mr, addr, TARGET_PAGE_SIZE);
368
            } while (c != 0);
369
        }
370
    }
371
    return 0;
372
}
373

    
374
#define ALIGN(x, y)  (((x)+(y)-1) & ~((y)-1))
375

    
376
/**
377
 * kvm_physical_sync_dirty_bitmap - Grab dirty bitmap from kernel space
378
 * This function updates qemu's dirty bitmap using
379
 * memory_region_set_dirty().  This means all bits are set
380
 * to dirty.
381
 *
382
 * @start_add: start of logged region.
383
 * @end_addr: end of logged region.
384
 */
385
static int kvm_physical_sync_dirty_bitmap(MemoryRegionSection *section)
386
{
387
    KVMState *s = kvm_state;
388
    unsigned long size, allocated_size = 0;
389
    KVMDirtyLog d;
390
    KVMSlot *mem;
391
    int ret = 0;
392
    target_phys_addr_t start_addr = section->offset_within_address_space;
393
    target_phys_addr_t end_addr = start_addr + section->size;
394

    
395
    d.dirty_bitmap = NULL;
396
    while (start_addr < end_addr) {
397
        mem = kvm_lookup_overlapping_slot(s, start_addr, end_addr);
398
        if (mem == NULL) {
399
            break;
400
        }
401

    
402
        /* XXX bad kernel interface alert
403
         * For dirty bitmap, kernel allocates array of size aligned to
404
         * bits-per-long.  But for case when the kernel is 64bits and
405
         * the userspace is 32bits, userspace can't align to the same
406
         * bits-per-long, since sizeof(long) is different between kernel
407
         * and user space.  This way, userspace will provide buffer which
408
         * may be 4 bytes less than the kernel will use, resulting in
409
         * userspace memory corruption (which is not detectable by valgrind
410
         * too, in most cases).
411
         * So for now, let's align to 64 instead of HOST_LONG_BITS here, in
412
         * a hope that sizeof(long) wont become >8 any time soon.
413
         */
414
        size = ALIGN(((mem->memory_size) >> TARGET_PAGE_BITS),
415
                     /*HOST_LONG_BITS*/ 64) / 8;
416
        if (!d.dirty_bitmap) {
417
            d.dirty_bitmap = g_malloc(size);
418
        } else if (size > allocated_size) {
419
            d.dirty_bitmap = g_realloc(d.dirty_bitmap, size);
420
        }
421
        allocated_size = size;
422
        memset(d.dirty_bitmap, 0, allocated_size);
423

    
424
        d.slot = mem->slot;
425

    
426
        if (kvm_vm_ioctl(s, KVM_GET_DIRTY_LOG, &d) == -1) {
427
            DPRINTF("ioctl failed %d\n", errno);
428
            ret = -1;
429
            break;
430
        }
431

    
432
        kvm_get_dirty_pages_log_range(section, d.dirty_bitmap);
433
        start_addr = mem->start_addr + mem->memory_size;
434
    }
435
    g_free(d.dirty_bitmap);
436

    
437
    return ret;
438
}
439

    
440
int kvm_coalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
441
{
442
    int ret = -ENOSYS;
443
    KVMState *s = kvm_state;
444

    
445
    if (s->coalesced_mmio) {
446
        struct kvm_coalesced_mmio_zone zone;
447

    
448
        zone.addr = start;
449
        zone.size = size;
450

    
451
        ret = kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
452
    }
453

    
454
    return ret;
455
}
456

    
457
int kvm_uncoalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
458
{
459
    int ret = -ENOSYS;
460
    KVMState *s = kvm_state;
461

    
462
    if (s->coalesced_mmio) {
463
        struct kvm_coalesced_mmio_zone zone;
464

    
465
        zone.addr = start;
466
        zone.size = size;
467

    
468
        ret = kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
469
    }
470

    
471
    return ret;
472
}
473

    
474
int kvm_check_extension(KVMState *s, unsigned int extension)
475
{
476
    int ret;
477

    
478
    ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, extension);
479
    if (ret < 0) {
480
        ret = 0;
481
    }
482

    
483
    return ret;
484
}
485

    
486
static int kvm_check_many_ioeventfds(void)
487
{
488
    /* Userspace can use ioeventfd for io notification.  This requires a host
489
     * that supports eventfd(2) and an I/O thread; since eventfd does not
490
     * support SIGIO it cannot interrupt the vcpu.
491
     *
492
     * Older kernels have a 6 device limit on the KVM io bus.  Find out so we
493
     * can avoid creating too many ioeventfds.
494
     */
495
#if defined(CONFIG_EVENTFD)
496
    int ioeventfds[7];
497
    int i, ret = 0;
498
    for (i = 0; i < ARRAY_SIZE(ioeventfds); i++) {
499
        ioeventfds[i] = eventfd(0, EFD_CLOEXEC);
500
        if (ioeventfds[i] < 0) {
501
            break;
502
        }
503
        ret = kvm_set_ioeventfd_pio_word(ioeventfds[i], 0, i, true);
504
        if (ret < 0) {
505
            close(ioeventfds[i]);
506
            break;
507
        }
508
    }
509

    
510
    /* Decide whether many devices are supported or not */
511
    ret = i == ARRAY_SIZE(ioeventfds);
512

    
513
    while (i-- > 0) {
514
        kvm_set_ioeventfd_pio_word(ioeventfds[i], 0, i, false);
515
        close(ioeventfds[i]);
516
    }
517
    return ret;
518
#else
519
    return 0;
520
#endif
521
}
522

    
523
static const KVMCapabilityInfo *
524
kvm_check_extension_list(KVMState *s, const KVMCapabilityInfo *list)
525
{
526
    while (list->name) {
527
        if (!kvm_check_extension(s, list->value)) {
528
            return list;
529
        }
530
        list++;
531
    }
532
    return NULL;
533
}
534

    
535
static void kvm_set_phys_mem(MemoryRegionSection *section, bool add)
536
{
537
    KVMState *s = kvm_state;
538
    KVMSlot *mem, old;
539
    int err;
540
    MemoryRegion *mr = section->mr;
541
    bool log_dirty = memory_region_is_logging(mr);
542
    target_phys_addr_t start_addr = section->offset_within_address_space;
543
    ram_addr_t size = section->size;
544
    void *ram = NULL;
545
    unsigned delta;
546

    
547
    /* kvm works in page size chunks, but the function may be called
548
       with sub-page size and unaligned start address. */
549
    delta = TARGET_PAGE_ALIGN(size) - size;
550
    if (delta > size) {
551
        return;
552
    }
553
    start_addr += delta;
554
    size -= delta;
555
    size &= TARGET_PAGE_MASK;
556
    if (!size || (start_addr & ~TARGET_PAGE_MASK)) {
557
        return;
558
    }
559

    
560
    if (!memory_region_is_ram(mr)) {
561
        return;
562
    }
563

    
564
    ram = memory_region_get_ram_ptr(mr) + section->offset_within_region + delta;
565

    
566
    while (1) {
567
        mem = kvm_lookup_overlapping_slot(s, start_addr, start_addr + size);
568
        if (!mem) {
569
            break;
570
        }
571

    
572
        if (add && start_addr >= mem->start_addr &&
573
            (start_addr + size <= mem->start_addr + mem->memory_size) &&
574
            (ram - start_addr == mem->ram - mem->start_addr)) {
575
            /* The new slot fits into the existing one and comes with
576
             * identical parameters - update flags and done. */
577
            kvm_slot_dirty_pages_log_change(mem, log_dirty);
578
            return;
579
        }
580

    
581
        old = *mem;
582

    
583
        if (mem->flags & KVM_MEM_LOG_DIRTY_PAGES) {
584
            kvm_physical_sync_dirty_bitmap(section);
585
        }
586

    
587
        /* unregister the overlapping slot */
588
        mem->memory_size = 0;
589
        err = kvm_set_user_memory_region(s, mem);
590
        if (err) {
591
            fprintf(stderr, "%s: error unregistering overlapping slot: %s\n",
592
                    __func__, strerror(-err));
593
            abort();
594
        }
595

    
596
        /* Workaround for older KVM versions: we can't join slots, even not by
597
         * unregistering the previous ones and then registering the larger
598
         * slot. We have to maintain the existing fragmentation. Sigh.
599
         *
600
         * This workaround assumes that the new slot starts at the same
601
         * address as the first existing one. If not or if some overlapping
602
         * slot comes around later, we will fail (not seen in practice so far)
603
         * - and actually require a recent KVM version. */
604
        if (s->broken_set_mem_region &&
605
            old.start_addr == start_addr && old.memory_size < size && add) {
606
            mem = kvm_alloc_slot(s);
607
            mem->memory_size = old.memory_size;
608
            mem->start_addr = old.start_addr;
609
            mem->ram = old.ram;
610
            mem->flags = kvm_mem_flags(s, log_dirty);
611

    
612
            err = kvm_set_user_memory_region(s, mem);
613
            if (err) {
614
                fprintf(stderr, "%s: error updating slot: %s\n", __func__,
615
                        strerror(-err));
616
                abort();
617
            }
618

    
619
            start_addr += old.memory_size;
620
            ram += old.memory_size;
621
            size -= old.memory_size;
622
            continue;
623
        }
624

    
625
        /* register prefix slot */
626
        if (old.start_addr < start_addr) {
627
            mem = kvm_alloc_slot(s);
628
            mem->memory_size = start_addr - old.start_addr;
629
            mem->start_addr = old.start_addr;
630
            mem->ram = old.ram;
631
            mem->flags =  kvm_mem_flags(s, log_dirty);
632

    
633
            err = kvm_set_user_memory_region(s, mem);
634
            if (err) {
635
                fprintf(stderr, "%s: error registering prefix slot: %s\n",
636
                        __func__, strerror(-err));
637
#ifdef TARGET_PPC
638
                fprintf(stderr, "%s: This is probably because your kernel's " \
639
                                "PAGE_SIZE is too big. Please try to use 4k " \
640
                                "PAGE_SIZE!\n", __func__);
641
#endif
642
                abort();
643
            }
644
        }
645

    
646
        /* register suffix slot */
647
        if (old.start_addr + old.memory_size > start_addr + size) {
648
            ram_addr_t size_delta;
649

    
650
            mem = kvm_alloc_slot(s);
651
            mem->start_addr = start_addr + size;
652
            size_delta = mem->start_addr - old.start_addr;
653
            mem->memory_size = old.memory_size - size_delta;
654
            mem->ram = old.ram + size_delta;
655
            mem->flags = kvm_mem_flags(s, log_dirty);
656

    
657
            err = kvm_set_user_memory_region(s, mem);
658
            if (err) {
659
                fprintf(stderr, "%s: error registering suffix slot: %s\n",
660
                        __func__, strerror(-err));
661
                abort();
662
            }
663
        }
664
    }
665

    
666
    /* in case the KVM bug workaround already "consumed" the new slot */
667
    if (!size) {
668
        return;
669
    }
670
    if (!add) {
671
        return;
672
    }
673
    mem = kvm_alloc_slot(s);
674
    mem->memory_size = size;
675
    mem->start_addr = start_addr;
676
    mem->ram = ram;
677
    mem->flags = kvm_mem_flags(s, log_dirty);
678

    
679
    err = kvm_set_user_memory_region(s, mem);
680
    if (err) {
681
        fprintf(stderr, "%s: error registering slot: %s\n", __func__,
682
                strerror(-err));
683
        abort();
684
    }
685
}
686

    
687
static void kvm_begin(MemoryListener *listener)
688
{
689
}
690

    
691
static void kvm_commit(MemoryListener *listener)
692
{
693
}
694

    
695
static void kvm_region_add(MemoryListener *listener,
696
                           MemoryRegionSection *section)
697
{
698
    kvm_set_phys_mem(section, true);
699
}
700

    
701
static void kvm_region_del(MemoryListener *listener,
702
                           MemoryRegionSection *section)
703
{
704
    kvm_set_phys_mem(section, false);
705
}
706

    
707
static void kvm_region_nop(MemoryListener *listener,
708
                           MemoryRegionSection *section)
709
{
710
}
711

    
712
static void kvm_log_sync(MemoryListener *listener,
713
                         MemoryRegionSection *section)
714
{
715
    int r;
716

    
717
    r = kvm_physical_sync_dirty_bitmap(section);
718
    if (r < 0) {
719
        abort();
720
    }
721
}
722

    
723
static void kvm_log_global_start(struct MemoryListener *listener)
724
{
725
    int r;
726

    
727
    r = kvm_set_migration_log(1);
728
    assert(r >= 0);
729
}
730

    
731
static void kvm_log_global_stop(struct MemoryListener *listener)
732
{
733
    int r;
734

    
735
    r = kvm_set_migration_log(0);
736
    assert(r >= 0);
737
}
738

    
739
static void kvm_mem_ioeventfd_add(MemoryRegionSection *section,
740
                                  bool match_data, uint64_t data, int fd)
741
{
742
    int r;
743

    
744
    assert(match_data && section->size == 4);
745

    
746
    r = kvm_set_ioeventfd_mmio_long(fd, section->offset_within_address_space,
747
                                    data, true);
748
    if (r < 0) {
749
        abort();
750
    }
751
}
752

    
753
static void kvm_mem_ioeventfd_del(MemoryRegionSection *section,
754
                                  bool match_data, uint64_t data, int fd)
755
{
756
    int r;
757

    
758
    r = kvm_set_ioeventfd_mmio_long(fd, section->offset_within_address_space,
759
                                    data, false);
760
    if (r < 0) {
761
        abort();
762
    }
763
}
764

    
765
static void kvm_io_ioeventfd_add(MemoryRegionSection *section,
766
                                 bool match_data, uint64_t data, int fd)
767
{
768
    int r;
769

    
770
    assert(match_data && section->size == 2);
771

    
772
    r = kvm_set_ioeventfd_pio_word(fd, section->offset_within_address_space,
773
                                   data, true);
774
    if (r < 0) {
775
        abort();
776
    }
777
}
778

    
779
static void kvm_io_ioeventfd_del(MemoryRegionSection *section,
780
                                 bool match_data, uint64_t data, int fd)
781

    
782
{
783
    int r;
784

    
785
    r = kvm_set_ioeventfd_pio_word(fd, section->offset_within_address_space,
786
                                   data, false);
787
    if (r < 0) {
788
        abort();
789
    }
790
}
791

    
792
static void kvm_eventfd_add(MemoryListener *listener,
793
                            MemoryRegionSection *section,
794
                            bool match_data, uint64_t data, int fd)
795
{
796
    if (section->address_space == get_system_memory()) {
797
        kvm_mem_ioeventfd_add(section, match_data, data, fd);
798
    } else {
799
        kvm_io_ioeventfd_add(section, match_data, data, fd);
800
    }
801
}
802

    
803
static void kvm_eventfd_del(MemoryListener *listener,
804
                            MemoryRegionSection *section,
805
                            bool match_data, uint64_t data, int fd)
806
{
807
    if (section->address_space == get_system_memory()) {
808
        kvm_mem_ioeventfd_del(section, match_data, data, fd);
809
    } else {
810
        kvm_io_ioeventfd_del(section, match_data, data, fd);
811
    }
812
}
813

    
814
static MemoryListener kvm_memory_listener = {
815
    .begin = kvm_begin,
816
    .commit = kvm_commit,
817
    .region_add = kvm_region_add,
818
    .region_del = kvm_region_del,
819
    .region_nop = kvm_region_nop,
820
    .log_start = kvm_log_start,
821
    .log_stop = kvm_log_stop,
822
    .log_sync = kvm_log_sync,
823
    .log_global_start = kvm_log_global_start,
824
    .log_global_stop = kvm_log_global_stop,
825
    .eventfd_add = kvm_eventfd_add,
826
    .eventfd_del = kvm_eventfd_del,
827
    .priority = 10,
828
};
829

    
830
static void kvm_handle_interrupt(CPUState *env, int mask)
831
{
832
    env->interrupt_request |= mask;
833

    
834
    if (!qemu_cpu_is_self(env)) {
835
        qemu_cpu_kick(env);
836
    }
837
}
838

    
839
int kvm_irqchip_set_irq(KVMState *s, int irq, int level)
840
{
841
    struct kvm_irq_level event;
842
    int ret;
843

    
844
    assert(kvm_irqchip_in_kernel());
845

    
846
    event.level = level;
847
    event.irq = irq;
848
    ret = kvm_vm_ioctl(s, s->irqchip_inject_ioctl, &event);
849
    if (ret < 0) {
850
        perror("kvm_set_irqchip_line");
851
        abort();
852
    }
853

    
854
    return (s->irqchip_inject_ioctl == KVM_IRQ_LINE) ? 1 : event.status;
855
}
856

    
857
#ifdef KVM_CAP_IRQ_ROUTING
858
static void set_gsi(KVMState *s, unsigned int gsi)
859
{
860
    assert(gsi < s->max_gsi);
861

    
862
    s->used_gsi_bitmap[gsi / 32] |= 1U << (gsi % 32);
863
}
864

    
865
static void kvm_init_irq_routing(KVMState *s)
866
{
867
    int gsi_count;
868

    
869
    gsi_count = kvm_check_extension(s, KVM_CAP_IRQ_ROUTING);
870
    if (gsi_count > 0) {
871
        unsigned int gsi_bits, i;
872

    
873
        /* Round up so we can search ints using ffs */
874
        gsi_bits = (gsi_count + 31) / 32;
875
        s->used_gsi_bitmap = g_malloc0(gsi_bits / 8);
876
        s->max_gsi = gsi_bits;
877

    
878
        /* Mark any over-allocated bits as already in use */
879
        for (i = gsi_count; i < gsi_bits; i++) {
880
            set_gsi(s, i);
881
        }
882
    }
883

    
884
    s->irq_routes = g_malloc0(sizeof(*s->irq_routes));
885
    s->nr_allocated_irq_routes = 0;
886

    
887
    kvm_arch_init_irq_routing(s);
888
}
889

    
890
static void kvm_add_routing_entry(KVMState *s,
891
                                  struct kvm_irq_routing_entry *entry)
892
{
893
    struct kvm_irq_routing_entry *new;
894
    int n, size;
895

    
896
    if (s->irq_routes->nr == s->nr_allocated_irq_routes) {
897
        n = s->nr_allocated_irq_routes * 2;
898
        if (n < 64) {
899
            n = 64;
900
        }
901
        size = sizeof(struct kvm_irq_routing);
902
        size += n * sizeof(*new);
903
        s->irq_routes = g_realloc(s->irq_routes, size);
904
        s->nr_allocated_irq_routes = n;
905
    }
906
    n = s->irq_routes->nr++;
907
    new = &s->irq_routes->entries[n];
908
    memset(new, 0, sizeof(*new));
909
    new->gsi = entry->gsi;
910
    new->type = entry->type;
911
    new->flags = entry->flags;
912
    new->u = entry->u;
913

    
914
    set_gsi(s, entry->gsi);
915
}
916

    
917
void kvm_irqchip_add_route(KVMState *s, int irq, int irqchip, int pin)
918
{
919
    struct kvm_irq_routing_entry e;
920

    
921
    e.gsi = irq;
922
    e.type = KVM_IRQ_ROUTING_IRQCHIP;
923
    e.flags = 0;
924
    e.u.irqchip.irqchip = irqchip;
925
    e.u.irqchip.pin = pin;
926
    kvm_add_routing_entry(s, &e);
927
}
928

    
929
int kvm_irqchip_commit_routes(KVMState *s)
930
{
931
    s->irq_routes->flags = 0;
932
    return kvm_vm_ioctl(s, KVM_SET_GSI_ROUTING, s->irq_routes);
933
}
934

    
935
#else /* !KVM_CAP_IRQ_ROUTING */
936

    
937
static void kvm_init_irq_routing(KVMState *s)
938
{
939
}
940
#endif /* !KVM_CAP_IRQ_ROUTING */
941

    
942
static int kvm_irqchip_create(KVMState *s)
943
{
944
    QemuOptsList *list = qemu_find_opts("machine");
945
    int ret;
946

    
947
    if (QTAILQ_EMPTY(&list->head) ||
948
        !qemu_opt_get_bool(QTAILQ_FIRST(&list->head),
949
                           "kernel_irqchip", false) ||
950
        !kvm_check_extension(s, KVM_CAP_IRQCHIP)) {
951
        return 0;
952
    }
953

    
954
    ret = kvm_vm_ioctl(s, KVM_CREATE_IRQCHIP);
955
    if (ret < 0) {
956
        fprintf(stderr, "Create kernel irqchip failed\n");
957
        return ret;
958
    }
959

    
960
    s->irqchip_inject_ioctl = KVM_IRQ_LINE;
961
    if (kvm_check_extension(s, KVM_CAP_IRQ_INJECT_STATUS)) {
962
        s->irqchip_inject_ioctl = KVM_IRQ_LINE_STATUS;
963
    }
964
    kvm_kernel_irqchip = true;
965

    
966
    kvm_init_irq_routing(s);
967

    
968
    return 0;
969
}
970

    
971
int kvm_init(void)
972
{
973
    static const char upgrade_note[] =
974
        "Please upgrade to at least kernel 2.6.29 or recent kvm-kmod\n"
975
        "(see http://sourceforge.net/projects/kvm).\n";
976
    KVMState *s;
977
    const KVMCapabilityInfo *missing_cap;
978
    int ret;
979
    int i;
980

    
981
    s = g_malloc0(sizeof(KVMState));
982

    
983
#ifdef KVM_CAP_SET_GUEST_DEBUG
984
    QTAILQ_INIT(&s->kvm_sw_breakpoints);
985
#endif
986
    for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
987
        s->slots[i].slot = i;
988
    }
989
    s->vmfd = -1;
990
    s->fd = qemu_open("/dev/kvm", O_RDWR);
991
    if (s->fd == -1) {
992
        fprintf(stderr, "Could not access KVM kernel module: %m\n");
993
        ret = -errno;
994
        goto err;
995
    }
996

    
997
    ret = kvm_ioctl(s, KVM_GET_API_VERSION, 0);
998
    if (ret < KVM_API_VERSION) {
999
        if (ret > 0) {
1000
            ret = -EINVAL;
1001
        }
1002
        fprintf(stderr, "kvm version too old\n");
1003
        goto err;
1004
    }
1005

    
1006
    if (ret > KVM_API_VERSION) {
1007
        ret = -EINVAL;
1008
        fprintf(stderr, "kvm version not supported\n");
1009
        goto err;
1010
    }
1011

    
1012
    s->vmfd = kvm_ioctl(s, KVM_CREATE_VM, 0);
1013
    if (s->vmfd < 0) {
1014
#ifdef TARGET_S390X
1015
        fprintf(stderr, "Please add the 'switch_amode' kernel parameter to "
1016
                        "your host kernel command line\n");
1017
#endif
1018
        ret = s->vmfd;
1019
        goto err;
1020
    }
1021

    
1022
    missing_cap = kvm_check_extension_list(s, kvm_required_capabilites);
1023
    if (!missing_cap) {
1024
        missing_cap =
1025
            kvm_check_extension_list(s, kvm_arch_required_capabilities);
1026
    }
1027
    if (missing_cap) {
1028
        ret = -EINVAL;
1029
        fprintf(stderr, "kvm does not support %s\n%s",
1030
                missing_cap->name, upgrade_note);
1031
        goto err;
1032
    }
1033

    
1034
    s->coalesced_mmio = kvm_check_extension(s, KVM_CAP_COALESCED_MMIO);
1035

    
1036
    s->broken_set_mem_region = 1;
1037
    ret = kvm_check_extension(s, KVM_CAP_JOIN_MEMORY_REGIONS_WORKS);
1038
    if (ret > 0) {
1039
        s->broken_set_mem_region = 0;
1040
    }
1041

    
1042
#ifdef KVM_CAP_VCPU_EVENTS
1043
    s->vcpu_events = kvm_check_extension(s, KVM_CAP_VCPU_EVENTS);
1044
#endif
1045

    
1046
    s->robust_singlestep =
1047
        kvm_check_extension(s, KVM_CAP_X86_ROBUST_SINGLESTEP);
1048

    
1049
#ifdef KVM_CAP_DEBUGREGS
1050
    s->debugregs = kvm_check_extension(s, KVM_CAP_DEBUGREGS);
1051
#endif
1052

    
1053
#ifdef KVM_CAP_XSAVE
1054
    s->xsave = kvm_check_extension(s, KVM_CAP_XSAVE);
1055
#endif
1056

    
1057
#ifdef KVM_CAP_XCRS
1058
    s->xcrs = kvm_check_extension(s, KVM_CAP_XCRS);
1059
#endif
1060

    
1061
    ret = kvm_arch_init(s);
1062
    if (ret < 0) {
1063
        goto err;
1064
    }
1065

    
1066
    ret = kvm_irqchip_create(s);
1067
    if (ret < 0) {
1068
        goto err;
1069
    }
1070

    
1071
    kvm_state = s;
1072
    memory_listener_register(&kvm_memory_listener, NULL);
1073

    
1074
    s->many_ioeventfds = kvm_check_many_ioeventfds();
1075

    
1076
    cpu_interrupt_handler = kvm_handle_interrupt;
1077

    
1078
    return 0;
1079

    
1080
err:
1081
    if (s) {
1082
        if (s->vmfd >= 0) {
1083
            close(s->vmfd);
1084
        }
1085
        if (s->fd != -1) {
1086
            close(s->fd);
1087
        }
1088
    }
1089
    g_free(s);
1090

    
1091
    return ret;
1092
}
1093

    
1094
static void kvm_handle_io(uint16_t port, void *data, int direction, int size,
1095
                          uint32_t count)
1096
{
1097
    int i;
1098
    uint8_t *ptr = data;
1099

    
1100
    for (i = 0; i < count; i++) {
1101
        if (direction == KVM_EXIT_IO_IN) {
1102
            switch (size) {
1103
            case 1:
1104
                stb_p(ptr, cpu_inb(port));
1105
                break;
1106
            case 2:
1107
                stw_p(ptr, cpu_inw(port));
1108
                break;
1109
            case 4:
1110
                stl_p(ptr, cpu_inl(port));
1111
                break;
1112
            }
1113
        } else {
1114
            switch (size) {
1115
            case 1:
1116
                cpu_outb(port, ldub_p(ptr));
1117
                break;
1118
            case 2:
1119
                cpu_outw(port, lduw_p(ptr));
1120
                break;
1121
            case 4:
1122
                cpu_outl(port, ldl_p(ptr));
1123
                break;
1124
            }
1125
        }
1126

    
1127
        ptr += size;
1128
    }
1129
}
1130

    
1131
static int kvm_handle_internal_error(CPUState *env, struct kvm_run *run)
1132
{
1133
    fprintf(stderr, "KVM internal error.");
1134
    if (kvm_check_extension(kvm_state, KVM_CAP_INTERNAL_ERROR_DATA)) {
1135
        int i;
1136

    
1137
        fprintf(stderr, " Suberror: %d\n", run->internal.suberror);
1138
        for (i = 0; i < run->internal.ndata; ++i) {
1139
            fprintf(stderr, "extra data[%d]: %"PRIx64"\n",
1140
                    i, (uint64_t)run->internal.data[i]);
1141
        }
1142
    } else {
1143
        fprintf(stderr, "\n");
1144
    }
1145
    if (run->internal.suberror == KVM_INTERNAL_ERROR_EMULATION) {
1146
        fprintf(stderr, "emulation failure\n");
1147
        if (!kvm_arch_stop_on_emulation_error(env)) {
1148
            cpu_dump_state(env, stderr, fprintf, CPU_DUMP_CODE);
1149
            return EXCP_INTERRUPT;
1150
        }
1151
    }
1152
    /* FIXME: Should trigger a qmp message to let management know
1153
     * something went wrong.
1154
     */
1155
    return -1;
1156
}
1157

    
1158
void kvm_flush_coalesced_mmio_buffer(void)
1159
{
1160
    KVMState *s = kvm_state;
1161

    
1162
    if (s->coalesced_flush_in_progress) {
1163
        return;
1164
    }
1165

    
1166
    s->coalesced_flush_in_progress = true;
1167

    
1168
    if (s->coalesced_mmio_ring) {
1169
        struct kvm_coalesced_mmio_ring *ring = s->coalesced_mmio_ring;
1170
        while (ring->first != ring->last) {
1171
            struct kvm_coalesced_mmio *ent;
1172

    
1173
            ent = &ring->coalesced_mmio[ring->first];
1174

    
1175
            cpu_physical_memory_write(ent->phys_addr, ent->data, ent->len);
1176
            smp_wmb();
1177
            ring->first = (ring->first + 1) % KVM_COALESCED_MMIO_MAX;
1178
        }
1179
    }
1180

    
1181
    s->coalesced_flush_in_progress = false;
1182
}
1183

    
1184
static void do_kvm_cpu_synchronize_state(void *_env)
1185
{
1186
    CPUState *env = _env;
1187

    
1188
    if (!env->kvm_vcpu_dirty) {
1189
        kvm_arch_get_registers(env);
1190
        env->kvm_vcpu_dirty = 1;
1191
    }
1192
}
1193

    
1194
void kvm_cpu_synchronize_state(CPUState *env)
1195
{
1196
    if (!env->kvm_vcpu_dirty) {
1197
        run_on_cpu(env, do_kvm_cpu_synchronize_state, env);
1198
    }
1199
}
1200

    
1201
void kvm_cpu_synchronize_post_reset(CPUState *env)
1202
{
1203
    kvm_arch_put_registers(env, KVM_PUT_RESET_STATE);
1204
    env->kvm_vcpu_dirty = 0;
1205
}
1206

    
1207
void kvm_cpu_synchronize_post_init(CPUState *env)
1208
{
1209
    kvm_arch_put_registers(env, KVM_PUT_FULL_STATE);
1210
    env->kvm_vcpu_dirty = 0;
1211
}
1212

    
1213
int kvm_cpu_exec(CPUState *env)
1214
{
1215
    struct kvm_run *run = env->kvm_run;
1216
    int ret, run_ret;
1217

    
1218
    DPRINTF("kvm_cpu_exec()\n");
1219

    
1220
    if (kvm_arch_process_async_events(env)) {
1221
        env->exit_request = 0;
1222
        return EXCP_HLT;
1223
    }
1224

    
1225
    do {
1226
        if (env->kvm_vcpu_dirty) {
1227
            kvm_arch_put_registers(env, KVM_PUT_RUNTIME_STATE);
1228
            env->kvm_vcpu_dirty = 0;
1229
        }
1230

    
1231
        kvm_arch_pre_run(env, run);
1232
        if (env->exit_request) {
1233
            DPRINTF("interrupt exit requested\n");
1234
            /*
1235
             * KVM requires us to reenter the kernel after IO exits to complete
1236
             * instruction emulation. This self-signal will ensure that we
1237
             * leave ASAP again.
1238
             */
1239
            qemu_cpu_kick_self();
1240
        }
1241
        qemu_mutex_unlock_iothread();
1242

    
1243
        run_ret = kvm_vcpu_ioctl(env, KVM_RUN, 0);
1244

    
1245
        qemu_mutex_lock_iothread();
1246
        kvm_arch_post_run(env, run);
1247

    
1248
        kvm_flush_coalesced_mmio_buffer();
1249

    
1250
        if (run_ret < 0) {
1251
            if (run_ret == -EINTR || run_ret == -EAGAIN) {
1252
                DPRINTF("io window exit\n");
1253
                ret = EXCP_INTERRUPT;
1254
                break;
1255
            }
1256
            fprintf(stderr, "error: kvm run failed %s\n",
1257
                    strerror(-run_ret));
1258
            abort();
1259
        }
1260

    
1261
        switch (run->exit_reason) {
1262
        case KVM_EXIT_IO:
1263
            DPRINTF("handle_io\n");
1264
            kvm_handle_io(run->io.port,
1265
                          (uint8_t *)run + run->io.data_offset,
1266
                          run->io.direction,
1267
                          run->io.size,
1268
                          run->io.count);
1269
            ret = 0;
1270
            break;
1271
        case KVM_EXIT_MMIO:
1272
            DPRINTF("handle_mmio\n");
1273
            cpu_physical_memory_rw(run->mmio.phys_addr,
1274
                                   run->mmio.data,
1275
                                   run->mmio.len,
1276
                                   run->mmio.is_write);
1277
            ret = 0;
1278
            break;
1279
        case KVM_EXIT_IRQ_WINDOW_OPEN:
1280
            DPRINTF("irq_window_open\n");
1281
            ret = EXCP_INTERRUPT;
1282
            break;
1283
        case KVM_EXIT_SHUTDOWN:
1284
            DPRINTF("shutdown\n");
1285
            qemu_system_reset_request();
1286
            ret = EXCP_INTERRUPT;
1287
            break;
1288
        case KVM_EXIT_UNKNOWN:
1289
            fprintf(stderr, "KVM: unknown exit, hardware reason %" PRIx64 "\n",
1290
                    (uint64_t)run->hw.hardware_exit_reason);
1291
            ret = -1;
1292
            break;
1293
        case KVM_EXIT_INTERNAL_ERROR:
1294
            ret = kvm_handle_internal_error(env, run);
1295
            break;
1296
        default:
1297
            DPRINTF("kvm_arch_handle_exit\n");
1298
            ret = kvm_arch_handle_exit(env, run);
1299
            break;
1300
        }
1301
    } while (ret == 0);
1302

    
1303
    if (ret < 0) {
1304
        cpu_dump_state(env, stderr, fprintf, CPU_DUMP_CODE);
1305
        vm_stop(RUN_STATE_INTERNAL_ERROR);
1306
    }
1307

    
1308
    env->exit_request = 0;
1309
    return ret;
1310
}
1311

    
1312
int kvm_ioctl(KVMState *s, int type, ...)
1313
{
1314
    int ret;
1315
    void *arg;
1316
    va_list ap;
1317

    
1318
    va_start(ap, type);
1319
    arg = va_arg(ap, void *);
1320
    va_end(ap);
1321

    
1322
    ret = ioctl(s->fd, type, arg);
1323
    if (ret == -1) {
1324
        ret = -errno;
1325
    }
1326
    return ret;
1327
}
1328

    
1329
int kvm_vm_ioctl(KVMState *s, int type, ...)
1330
{
1331
    int ret;
1332
    void *arg;
1333
    va_list ap;
1334

    
1335
    va_start(ap, type);
1336
    arg = va_arg(ap, void *);
1337
    va_end(ap);
1338

    
1339
    ret = ioctl(s->vmfd, type, arg);
1340
    if (ret == -1) {
1341
        ret = -errno;
1342
    }
1343
    return ret;
1344
}
1345

    
1346
int kvm_vcpu_ioctl(CPUState *env, int type, ...)
1347
{
1348
    int ret;
1349
    void *arg;
1350
    va_list ap;
1351

    
1352
    va_start(ap, type);
1353
    arg = va_arg(ap, void *);
1354
    va_end(ap);
1355

    
1356
    ret = ioctl(env->kvm_fd, type, arg);
1357
    if (ret == -1) {
1358
        ret = -errno;
1359
    }
1360
    return ret;
1361
}
1362

    
1363
int kvm_has_sync_mmu(void)
1364
{
1365
    return kvm_check_extension(kvm_state, KVM_CAP_SYNC_MMU);
1366
}
1367

    
1368
int kvm_has_vcpu_events(void)
1369
{
1370
    return kvm_state->vcpu_events;
1371
}
1372

    
1373
int kvm_has_robust_singlestep(void)
1374
{
1375
    return kvm_state->robust_singlestep;
1376
}
1377

    
1378
int kvm_has_debugregs(void)
1379
{
1380
    return kvm_state->debugregs;
1381
}
1382

    
1383
int kvm_has_xsave(void)
1384
{
1385
    return kvm_state->xsave;
1386
}
1387

    
1388
int kvm_has_xcrs(void)
1389
{
1390
    return kvm_state->xcrs;
1391
}
1392

    
1393
int kvm_has_many_ioeventfds(void)
1394
{
1395
    if (!kvm_enabled()) {
1396
        return 0;
1397
    }
1398
    return kvm_state->many_ioeventfds;
1399
}
1400

    
1401
int kvm_has_gsi_routing(void)
1402
{
1403
#ifdef KVM_CAP_IRQ_ROUTING
1404
    return kvm_check_extension(kvm_state, KVM_CAP_IRQ_ROUTING);
1405
#else
1406
    return false;
1407
#endif
1408
}
1409

    
1410
int kvm_allows_irq0_override(void)
1411
{
1412
    return !kvm_irqchip_in_kernel() || kvm_has_gsi_routing();
1413
}
1414

    
1415
void kvm_setup_guest_memory(void *start, size_t size)
1416
{
1417
    if (!kvm_has_sync_mmu()) {
1418
        int ret = qemu_madvise(start, size, QEMU_MADV_DONTFORK);
1419

    
1420
        if (ret) {
1421
            perror("qemu_madvise");
1422
            fprintf(stderr,
1423
                    "Need MADV_DONTFORK in absence of synchronous KVM MMU\n");
1424
            exit(1);
1425
        }
1426
    }
1427
}
1428

    
1429
#ifdef KVM_CAP_SET_GUEST_DEBUG
1430
struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *env,
1431
                                                 target_ulong pc)
1432
{
1433
    struct kvm_sw_breakpoint *bp;
1434

    
1435
    QTAILQ_FOREACH(bp, &env->kvm_state->kvm_sw_breakpoints, entry) {
1436
        if (bp->pc == pc) {
1437
            return bp;
1438
        }
1439
    }
1440
    return NULL;
1441
}
1442

    
1443
int kvm_sw_breakpoints_active(CPUState *env)
1444
{
1445
    return !QTAILQ_EMPTY(&env->kvm_state->kvm_sw_breakpoints);
1446
}
1447

    
1448
struct kvm_set_guest_debug_data {
1449
    struct kvm_guest_debug dbg;
1450
    CPUState *env;
1451
    int err;
1452
};
1453

    
1454
static void kvm_invoke_set_guest_debug(void *data)
1455
{
1456
    struct kvm_set_guest_debug_data *dbg_data = data;
1457
    CPUState *env = dbg_data->env;
1458

    
1459
    dbg_data->err = kvm_vcpu_ioctl(env, KVM_SET_GUEST_DEBUG, &dbg_data->dbg);
1460
}
1461

    
1462
int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
1463
{
1464
    struct kvm_set_guest_debug_data data;
1465

    
1466
    data.dbg.control = reinject_trap;
1467

    
1468
    if (env->singlestep_enabled) {
1469
        data.dbg.control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP;
1470
    }
1471
    kvm_arch_update_guest_debug(env, &data.dbg);
1472
    data.env = env;
1473

    
1474
    run_on_cpu(env, kvm_invoke_set_guest_debug, &data);
1475
    return data.err;
1476
}
1477

    
1478
int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
1479
                          target_ulong len, int type)
1480
{
1481
    struct kvm_sw_breakpoint *bp;
1482
    CPUState *env;
1483
    int err;
1484

    
1485
    if (type == GDB_BREAKPOINT_SW) {
1486
        bp = kvm_find_sw_breakpoint(current_env, addr);
1487
        if (bp) {
1488
            bp->use_count++;
1489
            return 0;
1490
        }
1491

    
1492
        bp = g_malloc(sizeof(struct kvm_sw_breakpoint));
1493
        if (!bp) {
1494
            return -ENOMEM;
1495
        }
1496

    
1497
        bp->pc = addr;
1498
        bp->use_count = 1;
1499
        err = kvm_arch_insert_sw_breakpoint(current_env, bp);
1500
        if (err) {
1501
            g_free(bp);
1502
            return err;
1503
        }
1504

    
1505
        QTAILQ_INSERT_HEAD(&current_env->kvm_state->kvm_sw_breakpoints,
1506
                          bp, entry);
1507
    } else {
1508
        err = kvm_arch_insert_hw_breakpoint(addr, len, type);
1509
        if (err) {
1510
            return err;
1511
        }
1512
    }
1513

    
1514
    for (env = first_cpu; env != NULL; env = env->next_cpu) {
1515
        err = kvm_update_guest_debug(env, 0);
1516
        if (err) {
1517
            return err;
1518
        }
1519
    }
1520
    return 0;
1521
}
1522

    
1523
int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
1524
                          target_ulong len, int type)
1525
{
1526
    struct kvm_sw_breakpoint *bp;
1527
    CPUState *env;
1528
    int err;
1529

    
1530
    if (type == GDB_BREAKPOINT_SW) {
1531
        bp = kvm_find_sw_breakpoint(current_env, addr);
1532
        if (!bp) {
1533
            return -ENOENT;
1534
        }
1535

    
1536
        if (bp->use_count > 1) {
1537
            bp->use_count--;
1538
            return 0;
1539
        }
1540

    
1541
        err = kvm_arch_remove_sw_breakpoint(current_env, bp);
1542
        if (err) {
1543
            return err;
1544
        }
1545

    
1546
        QTAILQ_REMOVE(&current_env->kvm_state->kvm_sw_breakpoints, bp, entry);
1547
        g_free(bp);
1548
    } else {
1549
        err = kvm_arch_remove_hw_breakpoint(addr, len, type);
1550
        if (err) {
1551
            return err;
1552
        }
1553
    }
1554

    
1555
    for (env = first_cpu; env != NULL; env = env->next_cpu) {
1556
        err = kvm_update_guest_debug(env, 0);
1557
        if (err) {
1558
            return err;
1559
        }
1560
    }
1561
    return 0;
1562
}
1563

    
1564
void kvm_remove_all_breakpoints(CPUState *current_env)
1565
{
1566
    struct kvm_sw_breakpoint *bp, *next;
1567
    KVMState *s = current_env->kvm_state;
1568
    CPUState *env;
1569

    
1570
    QTAILQ_FOREACH_SAFE(bp, &s->kvm_sw_breakpoints, entry, next) {
1571
        if (kvm_arch_remove_sw_breakpoint(current_env, bp) != 0) {
1572
            /* Try harder to find a CPU that currently sees the breakpoint. */
1573
            for (env = first_cpu; env != NULL; env = env->next_cpu) {
1574
                if (kvm_arch_remove_sw_breakpoint(env, bp) == 0) {
1575
                    break;
1576
                }
1577
            }
1578
        }
1579
    }
1580
    kvm_arch_remove_all_hw_breakpoints();
1581

    
1582
    for (env = first_cpu; env != NULL; env = env->next_cpu) {
1583
        kvm_update_guest_debug(env, 0);
1584
    }
1585
}
1586

    
1587
#else /* !KVM_CAP_SET_GUEST_DEBUG */
1588

    
1589
int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
1590
{
1591
    return -EINVAL;
1592
}
1593

    
1594
int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
1595
                          target_ulong len, int type)
1596
{
1597
    return -EINVAL;
1598
}
1599

    
1600
int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
1601
                          target_ulong len, int type)
1602
{
1603
    return -EINVAL;
1604
}
1605

    
1606
void kvm_remove_all_breakpoints(CPUState *current_env)
1607
{
1608
}
1609
#endif /* !KVM_CAP_SET_GUEST_DEBUG */
1610

    
1611
int kvm_set_signal_mask(CPUState *env, const sigset_t *sigset)
1612
{
1613
    struct kvm_signal_mask *sigmask;
1614
    int r;
1615

    
1616
    if (!sigset) {
1617
        return kvm_vcpu_ioctl(env, KVM_SET_SIGNAL_MASK, NULL);
1618
    }
1619

    
1620
    sigmask = g_malloc(sizeof(*sigmask) + sizeof(*sigset));
1621

    
1622
    sigmask->len = 8;
1623
    memcpy(sigmask->sigset, sigset, sizeof(*sigset));
1624
    r = kvm_vcpu_ioctl(env, KVM_SET_SIGNAL_MASK, sigmask);
1625
    g_free(sigmask);
1626

    
1627
    return r;
1628
}
1629

    
1630
int kvm_set_ioeventfd_mmio_long(int fd, uint32_t addr, uint32_t val, bool assign)
1631
{
1632
    int ret;
1633
    struct kvm_ioeventfd iofd;
1634

    
1635
    iofd.datamatch = val;
1636
    iofd.addr = addr;
1637
    iofd.len = 4;
1638
    iofd.flags = KVM_IOEVENTFD_FLAG_DATAMATCH;
1639
    iofd.fd = fd;
1640

    
1641
    if (!kvm_enabled()) {
1642
        return -ENOSYS;
1643
    }
1644

    
1645
    if (!assign) {
1646
        iofd.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
1647
    }
1648

    
1649
    ret = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &iofd);
1650

    
1651
    if (ret < 0) {
1652
        return -errno;
1653
    }
1654

    
1655
    return 0;
1656
}
1657

    
1658
int kvm_set_ioeventfd_pio_word(int fd, uint16_t addr, uint16_t val, bool assign)
1659
{
1660
    struct kvm_ioeventfd kick = {
1661
        .datamatch = val,
1662
        .addr = addr,
1663
        .len = 2,
1664
        .flags = KVM_IOEVENTFD_FLAG_DATAMATCH | KVM_IOEVENTFD_FLAG_PIO,
1665
        .fd = fd,
1666
    };
1667
    int r;
1668
    if (!kvm_enabled()) {
1669
        return -ENOSYS;
1670
    }
1671
    if (!assign) {
1672
        kick.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
1673
    }
1674
    r = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &kick);
1675
    if (r < 0) {
1676
        return r;
1677
    }
1678
    return 0;
1679
}
1680

    
1681
int kvm_on_sigbus_vcpu(CPUState *env, int code, void *addr)
1682
{
1683
    return kvm_arch_on_sigbus_vcpu(env, code, addr);
1684
}
1685

    
1686
int kvm_on_sigbus(int code, void *addr)
1687
{
1688
    return kvm_arch_on_sigbus(code, addr);
1689
}