Statistics
| Branch: | Revision:

root / kvm-all.c @ caa5af0f

History | View | Annotate | Download (27.2 kB)

1
/*
2
 * QEMU KVM support
3
 *
4
 * Copyright IBM, Corp. 2008
5
 *           Red Hat, Inc. 2008
6
 *
7
 * Authors:
8
 *  Anthony Liguori   <aliguori@us.ibm.com>
9
 *  Glauber Costa     <gcosta@redhat.com>
10
 *
11
 * This work is licensed under the terms of the GNU GPL, version 2 or later.
12
 * See the COPYING file in the top-level directory.
13
 *
14
 */
15

    
16
#include <sys/types.h>
17
#include <sys/ioctl.h>
18
#include <sys/mman.h>
19
#include <stdarg.h>
20

    
21
#include <linux/kvm.h>
22

    
23
#include "qemu-common.h"
24
#include "sysemu.h"
25
#include "hw/hw.h"
26
#include "gdbstub.h"
27
#include "kvm.h"
28

    
29
/* KVM uses PAGE_SIZE in it's definition of COALESCED_MMIO_MAX */
30
#define PAGE_SIZE TARGET_PAGE_SIZE
31

    
32
//#define DEBUG_KVM
33

    
34
#ifdef DEBUG_KVM
35
#define dprintf(fmt, ...) \
36
    do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
37
#else
38
#define dprintf(fmt, ...) \
39
    do { } while (0)
40
#endif
41

    
42
typedef struct KVMSlot
43
{
44
    target_phys_addr_t start_addr;
45
    ram_addr_t memory_size;
46
    ram_addr_t phys_offset;
47
    int slot;
48
    int flags;
49
} KVMSlot;
50

    
51
typedef struct kvm_dirty_log KVMDirtyLog;
52

    
53
int kvm_allowed = 0;
54

    
55
struct KVMState
56
{
57
    KVMSlot slots[32];
58
    int fd;
59
    int vmfd;
60
    int regs_modified;
61
    int coalesced_mmio;
62
    int broken_set_mem_region;
63
    int migration_log;
64
#ifdef KVM_CAP_SET_GUEST_DEBUG
65
    struct kvm_sw_breakpoint_head kvm_sw_breakpoints;
66
#endif
67
    int irqchip_in_kernel;
68
    int pit_in_kernel;
69
};
70

    
71
static KVMState *kvm_state;
72

    
73
static KVMSlot *kvm_alloc_slot(KVMState *s)
74
{
75
    int i;
76

    
77
    for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
78
        /* KVM private memory slots */
79
        if (i >= 8 && i < 12)
80
            continue;
81
        if (s->slots[i].memory_size == 0)
82
            return &s->slots[i];
83
    }
84

    
85
    fprintf(stderr, "%s: no free slot available\n", __func__);
86
    abort();
87
}
88

    
89
static KVMSlot *kvm_lookup_matching_slot(KVMState *s,
90
                                         target_phys_addr_t start_addr,
91
                                         target_phys_addr_t end_addr)
92
{
93
    int i;
94

    
95
    for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
96
        KVMSlot *mem = &s->slots[i];
97

    
98
        if (start_addr == mem->start_addr &&
99
            end_addr == mem->start_addr + mem->memory_size) {
100
            return mem;
101
        }
102
    }
103

    
104
    return NULL;
105
}
106

    
107
/*
108
 * Find overlapping slot with lowest start address
109
 */
110
static KVMSlot *kvm_lookup_overlapping_slot(KVMState *s,
111
                                            target_phys_addr_t start_addr,
112
                                            target_phys_addr_t end_addr)
113
{
114
    KVMSlot *found = NULL;
115
    int i;
116

    
117
    for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
118
        KVMSlot *mem = &s->slots[i];
119

    
120
        if (mem->memory_size == 0 ||
121
            (found && found->start_addr < mem->start_addr)) {
122
            continue;
123
        }
124

    
125
        if (end_addr > mem->start_addr &&
126
            start_addr < mem->start_addr + mem->memory_size) {
127
            found = mem;
128
        }
129
    }
130

    
131
    return found;
132
}
133

    
134
static int kvm_set_user_memory_region(KVMState *s, KVMSlot *slot)
135
{
136
    struct kvm_userspace_memory_region mem;
137

    
138
    mem.slot = slot->slot;
139
    mem.guest_phys_addr = slot->start_addr;
140
    mem.memory_size = slot->memory_size;
141
    mem.userspace_addr = (unsigned long)qemu_get_ram_ptr(slot->phys_offset);
142
    mem.flags = slot->flags;
143
    if (s->migration_log) {
144
        mem.flags |= KVM_MEM_LOG_DIRTY_PAGES;
145
    }
146
    return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
147
}
148

    
149
static void kvm_reset_vcpu(void *opaque)
150
{
151
    CPUState *env = opaque;
152

    
153
    kvm_arch_reset_vcpu(env);
154
    if (kvm_arch_put_registers(env)) {
155
        fprintf(stderr, "Fatal: kvm vcpu reset failed\n");
156
        abort();
157
    }
158
}
159

    
160
int kvm_irqchip_in_kernel(void)
161
{
162
    return kvm_state->irqchip_in_kernel;
163
}
164

    
165
int kvm_pit_in_kernel(void)
166
{
167
    return kvm_state->pit_in_kernel;
168
}
169

    
170

    
171
int kvm_init_vcpu(CPUState *env)
172
{
173
    KVMState *s = kvm_state;
174
    long mmap_size;
175
    int ret;
176

    
177
    dprintf("kvm_init_vcpu\n");
178

    
179
    ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, env->cpu_index);
180
    if (ret < 0) {
181
        dprintf("kvm_create_vcpu failed\n");
182
        goto err;
183
    }
184

    
185
    env->kvm_fd = ret;
186
    env->kvm_state = s;
187

    
188
    mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
189
    if (mmap_size < 0) {
190
        dprintf("KVM_GET_VCPU_MMAP_SIZE failed\n");
191
        goto err;
192
    }
193

    
194
    env->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
195
                        env->kvm_fd, 0);
196
    if (env->kvm_run == MAP_FAILED) {
197
        ret = -errno;
198
        dprintf("mmap'ing vcpu state failed\n");
199
        goto err;
200
    }
201

    
202
    ret = kvm_arch_init_vcpu(env);
203
    if (ret == 0) {
204
        qemu_register_reset(kvm_reset_vcpu, env);
205
        kvm_arch_reset_vcpu(env);
206
        ret = kvm_arch_put_registers(env);
207
    }
208
err:
209
    return ret;
210
}
211

    
212
/*
213
 * dirty pages logging control
214
 */
215
static int kvm_dirty_pages_log_change(target_phys_addr_t phys_addr,
216
                                      ram_addr_t size, int flags, int mask)
217
{
218
    KVMState *s = kvm_state;
219
    KVMSlot *mem = kvm_lookup_matching_slot(s, phys_addr, phys_addr + size);
220
    int old_flags;
221

    
222
    if (mem == NULL)  {
223
            fprintf(stderr, "BUG: %s: invalid parameters " TARGET_FMT_plx "-"
224
                    TARGET_FMT_plx "\n", __func__, phys_addr,
225
                    (target_phys_addr_t)(phys_addr + size - 1));
226
            return -EINVAL;
227
    }
228

    
229
    old_flags = mem->flags;
230

    
231
    flags = (mem->flags & ~mask) | flags;
232
    mem->flags = flags;
233

    
234
    /* If nothing changed effectively, no need to issue ioctl */
235
    if (s->migration_log) {
236
        flags |= KVM_MEM_LOG_DIRTY_PAGES;
237
    }
238
    if (flags == old_flags) {
239
            return 0;
240
    }
241

    
242
    return kvm_set_user_memory_region(s, mem);
243
}
244

    
245
int kvm_log_start(target_phys_addr_t phys_addr, ram_addr_t size)
246
{
247
        return kvm_dirty_pages_log_change(phys_addr, size,
248
                                          KVM_MEM_LOG_DIRTY_PAGES,
249
                                          KVM_MEM_LOG_DIRTY_PAGES);
250
}
251

    
252
int kvm_log_stop(target_phys_addr_t phys_addr, ram_addr_t size)
253
{
254
        return kvm_dirty_pages_log_change(phys_addr, size,
255
                                          0,
256
                                          KVM_MEM_LOG_DIRTY_PAGES);
257
}
258

    
259
int kvm_set_migration_log(int enable)
260
{
261
    KVMState *s = kvm_state;
262
    KVMSlot *mem;
263
    int i, err;
264

    
265
    s->migration_log = enable;
266

    
267
    for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
268
        mem = &s->slots[i];
269

    
270
        if (!!(mem->flags & KVM_MEM_LOG_DIRTY_PAGES) == enable) {
271
            continue;
272
        }
273
        err = kvm_set_user_memory_region(s, mem);
274
        if (err) {
275
            return err;
276
        }
277
    }
278
    return 0;
279
}
280

    
281
static int test_le_bit(unsigned long nr, unsigned char *addr)
282
{
283
    return (addr[nr >> 3] >> (nr & 7)) & 1;
284
}
285

    
286
/**
287
 * kvm_physical_sync_dirty_bitmap - Grab dirty bitmap from kernel space
288
 * This function updates qemu's dirty bitmap using cpu_physical_memory_set_dirty().
289
 * This means all bits are set to dirty.
290
 *
291
 * @start_add: start of logged region.
292
 * @end_addr: end of logged region.
293
 */
294
int kvm_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,
295
                                   target_phys_addr_t end_addr)
296
{
297
    KVMState *s = kvm_state;
298
    unsigned long size, allocated_size = 0;
299
    target_phys_addr_t phys_addr;
300
    ram_addr_t addr;
301
    KVMDirtyLog d;
302
    KVMSlot *mem;
303
    int ret = 0;
304

    
305
    d.dirty_bitmap = NULL;
306
    while (start_addr < end_addr) {
307
        mem = kvm_lookup_overlapping_slot(s, start_addr, end_addr);
308
        if (mem == NULL) {
309
            break;
310
        }
311

    
312
        size = ((mem->memory_size >> TARGET_PAGE_BITS) + 7) / 8;
313
        if (!d.dirty_bitmap) {
314
            d.dirty_bitmap = qemu_malloc(size);
315
        } else if (size > allocated_size) {
316
            d.dirty_bitmap = qemu_realloc(d.dirty_bitmap, size);
317
        }
318
        allocated_size = size;
319
        memset(d.dirty_bitmap, 0, allocated_size);
320

    
321
        d.slot = mem->slot;
322

    
323
        if (kvm_vm_ioctl(s, KVM_GET_DIRTY_LOG, &d) == -1) {
324
            dprintf("ioctl failed %d\n", errno);
325
            ret = -1;
326
            break;
327
        }
328

    
329
        for (phys_addr = mem->start_addr, addr = mem->phys_offset;
330
             phys_addr < mem->start_addr + mem->memory_size;
331
             phys_addr += TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
332
            unsigned char *bitmap = (unsigned char *)d.dirty_bitmap;
333
            unsigned nr = (phys_addr - mem->start_addr) >> TARGET_PAGE_BITS;
334

    
335
            if (test_le_bit(nr, bitmap)) {
336
                cpu_physical_memory_set_dirty(addr);
337
            }
338
        }
339
        start_addr = phys_addr;
340
    }
341
    qemu_free(d.dirty_bitmap);
342

    
343
    return ret;
344
}
345

    
346
int kvm_coalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
347
{
348
    int ret = -ENOSYS;
349
#ifdef KVM_CAP_COALESCED_MMIO
350
    KVMState *s = kvm_state;
351

    
352
    if (s->coalesced_mmio) {
353
        struct kvm_coalesced_mmio_zone zone;
354

    
355
        zone.addr = start;
356
        zone.size = size;
357

    
358
        ret = kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
359
    }
360
#endif
361

    
362
    return ret;
363
}
364

    
365
int kvm_uncoalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
366
{
367
    int ret = -ENOSYS;
368
#ifdef KVM_CAP_COALESCED_MMIO
369
    KVMState *s = kvm_state;
370

    
371
    if (s->coalesced_mmio) {
372
        struct kvm_coalesced_mmio_zone zone;
373

    
374
        zone.addr = start;
375
        zone.size = size;
376

    
377
        ret = kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
378
    }
379
#endif
380

    
381
    return ret;
382
}
383

    
384
int kvm_check_extension(KVMState *s, unsigned int extension)
385
{
386
    int ret;
387

    
388
    ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, extension);
389
    if (ret < 0) {
390
        ret = 0;
391
    }
392

    
393
    return ret;
394
}
395

    
396
int kvm_init(int smp_cpus)
397
{
398
    static const char upgrade_note[] =
399
        "Please upgrade to at least kernel 2.6.29 or recent kvm-kmod\n"
400
        "(see http://sourceforge.net/projects/kvm).\n";
401
    KVMState *s;
402
    int ret;
403
    int i;
404

    
405
    if (smp_cpus > 1) {
406
        fprintf(stderr, "No SMP KVM support, use '-smp 1'\n");
407
        return -EINVAL;
408
    }
409

    
410
    s = qemu_mallocz(sizeof(KVMState));
411

    
412
#ifdef KVM_CAP_SET_GUEST_DEBUG
413
    QTAILQ_INIT(&s->kvm_sw_breakpoints);
414
#endif
415
    for (i = 0; i < ARRAY_SIZE(s->slots); i++)
416
        s->slots[i].slot = i;
417

    
418
    s->vmfd = -1;
419
    s->fd = open("/dev/kvm", O_RDWR);
420
    if (s->fd == -1) {
421
        fprintf(stderr, "Could not access KVM kernel module: %m\n");
422
        ret = -errno;
423
        goto err;
424
    }
425

    
426
    ret = kvm_ioctl(s, KVM_GET_API_VERSION, 0);
427
    if (ret < KVM_API_VERSION) {
428
        if (ret > 0)
429
            ret = -EINVAL;
430
        fprintf(stderr, "kvm version too old\n");
431
        goto err;
432
    }
433

    
434
    if (ret > KVM_API_VERSION) {
435
        ret = -EINVAL;
436
        fprintf(stderr, "kvm version not supported\n");
437
        goto err;
438
    }
439

    
440
    s->vmfd = kvm_ioctl(s, KVM_CREATE_VM, 0);
441
    if (s->vmfd < 0)
442
        goto err;
443

    
444
    /* initially, KVM allocated its own memory and we had to jump through
445
     * hooks to make phys_ram_base point to this.  Modern versions of KVM
446
     * just use a user allocated buffer so we can use regular pages
447
     * unmodified.  Make sure we have a sufficiently modern version of KVM.
448
     */
449
    if (!kvm_check_extension(s, KVM_CAP_USER_MEMORY)) {
450
        ret = -EINVAL;
451
        fprintf(stderr, "kvm does not support KVM_CAP_USER_MEMORY\n%s",
452
                upgrade_note);
453
        goto err;
454
    }
455

    
456
    /* There was a nasty bug in < kvm-80 that prevents memory slots from being
457
     * destroyed properly.  Since we rely on this capability, refuse to work
458
     * with any kernel without this capability. */
459
    if (!kvm_check_extension(s, KVM_CAP_DESTROY_MEMORY_REGION_WORKS)) {
460
        ret = -EINVAL;
461

    
462
        fprintf(stderr,
463
                "KVM kernel module broken (DESTROY_MEMORY_REGION).\n%s",
464
                upgrade_note);
465
        goto err;
466
    }
467

    
468
#ifdef KVM_CAP_COALESCED_MMIO
469
    s->coalesced_mmio = kvm_check_extension(s, KVM_CAP_COALESCED_MMIO);
470
#else
471
    s->coalesced_mmio = 0;
472
#endif
473

    
474
    s->broken_set_mem_region = 1;
475
#ifdef KVM_CAP_JOIN_MEMORY_REGIONS_WORKS
476
    ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, KVM_CAP_JOIN_MEMORY_REGIONS_WORKS);
477
    if (ret > 0) {
478
        s->broken_set_mem_region = 0;
479
    }
480
#endif
481

    
482
    ret = kvm_arch_init(s, smp_cpus);
483
    if (ret < 0)
484
        goto err;
485

    
486
    kvm_state = s;
487

    
488
    return 0;
489

    
490
err:
491
    if (s) {
492
        if (s->vmfd != -1)
493
            close(s->vmfd);
494
        if (s->fd != -1)
495
            close(s->fd);
496
    }
497
    qemu_free(s);
498

    
499
    return ret;
500
}
501

    
502
static int kvm_handle_io(uint16_t port, void *data, int direction, int size,
503
                         uint32_t count)
504
{
505
    int i;
506
    uint8_t *ptr = data;
507

    
508
    for (i = 0; i < count; i++) {
509
        if (direction == KVM_EXIT_IO_IN) {
510
            switch (size) {
511
            case 1:
512
                stb_p(ptr, cpu_inb(port));
513
                break;
514
            case 2:
515
                stw_p(ptr, cpu_inw(port));
516
                break;
517
            case 4:
518
                stl_p(ptr, cpu_inl(port));
519
                break;
520
            }
521
        } else {
522
            switch (size) {
523
            case 1:
524
                cpu_outb(port, ldub_p(ptr));
525
                break;
526
            case 2:
527
                cpu_outw(port, lduw_p(ptr));
528
                break;
529
            case 4:
530
                cpu_outl(port, ldl_p(ptr));
531
                break;
532
            }
533
        }
534

    
535
        ptr += size;
536
    }
537

    
538
    return 1;
539
}
540

    
541
static void kvm_run_coalesced_mmio(CPUState *env, struct kvm_run *run)
542
{
543
#ifdef KVM_CAP_COALESCED_MMIO
544
    KVMState *s = kvm_state;
545
    if (s->coalesced_mmio) {
546
        struct kvm_coalesced_mmio_ring *ring;
547

    
548
        ring = (void *)run + (s->coalesced_mmio * TARGET_PAGE_SIZE);
549
        while (ring->first != ring->last) {
550
            struct kvm_coalesced_mmio *ent;
551

    
552
            ent = &ring->coalesced_mmio[ring->first];
553

    
554
            cpu_physical_memory_write(ent->phys_addr, ent->data, ent->len);
555
            /* FIXME smp_wmb() */
556
            ring->first = (ring->first + 1) % KVM_COALESCED_MMIO_MAX;
557
        }
558
    }
559
#endif
560
}
561

    
562
void kvm_cpu_synchronize_state(CPUState *env)
563
{
564
    if (!env->kvm_state->regs_modified) {
565
        kvm_arch_get_registers(env);
566
        env->kvm_state->regs_modified = 1;
567
    }
568
}
569

    
570
int kvm_cpu_exec(CPUState *env)
571
{
572
    struct kvm_run *run = env->kvm_run;
573
    int ret;
574

    
575
    dprintf("kvm_cpu_exec()\n");
576

    
577
    do {
578
        if (env->exit_request) {
579
            dprintf("interrupt exit requested\n");
580
            ret = 0;
581
            break;
582
        }
583

    
584
        if (env->kvm_state->regs_modified) {
585
            kvm_arch_put_registers(env);
586
            env->kvm_state->regs_modified = 0;
587
        }
588

    
589
        kvm_arch_pre_run(env, run);
590
        qemu_mutex_unlock_iothread();
591
        ret = kvm_vcpu_ioctl(env, KVM_RUN, 0);
592
        qemu_mutex_lock_iothread();
593
        kvm_arch_post_run(env, run);
594

    
595
        if (ret == -EINTR || ret == -EAGAIN) {
596
            dprintf("io window exit\n");
597
            ret = 0;
598
            break;
599
        }
600

    
601
        if (ret < 0) {
602
            dprintf("kvm run failed %s\n", strerror(-ret));
603
            abort();
604
        }
605

    
606
        kvm_run_coalesced_mmio(env, run);
607

    
608
        ret = 0; /* exit loop */
609
        switch (run->exit_reason) {
610
        case KVM_EXIT_IO:
611
            dprintf("handle_io\n");
612
            ret = kvm_handle_io(run->io.port,
613
                                (uint8_t *)run + run->io.data_offset,
614
                                run->io.direction,
615
                                run->io.size,
616
                                run->io.count);
617
            break;
618
        case KVM_EXIT_MMIO:
619
            dprintf("handle_mmio\n");
620
            cpu_physical_memory_rw(run->mmio.phys_addr,
621
                                   run->mmio.data,
622
                                   run->mmio.len,
623
                                   run->mmio.is_write);
624
            ret = 1;
625
            break;
626
        case KVM_EXIT_IRQ_WINDOW_OPEN:
627
            dprintf("irq_window_open\n");
628
            break;
629
        case KVM_EXIT_SHUTDOWN:
630
            dprintf("shutdown\n");
631
            qemu_system_reset_request();
632
            ret = 1;
633
            break;
634
        case KVM_EXIT_UNKNOWN:
635
            dprintf("kvm_exit_unknown\n");
636
            break;
637
        case KVM_EXIT_FAIL_ENTRY:
638
            dprintf("kvm_exit_fail_entry\n");
639
            break;
640
        case KVM_EXIT_EXCEPTION:
641
            dprintf("kvm_exit_exception\n");
642
            break;
643
        case KVM_EXIT_DEBUG:
644
            dprintf("kvm_exit_debug\n");
645
#ifdef KVM_CAP_SET_GUEST_DEBUG
646
            if (kvm_arch_debug(&run->debug.arch)) {
647
                gdb_set_stop_cpu(env);
648
                vm_stop(EXCP_DEBUG);
649
                env->exception_index = EXCP_DEBUG;
650
                return 0;
651
            }
652
            /* re-enter, this exception was guest-internal */
653
            ret = 1;
654
#endif /* KVM_CAP_SET_GUEST_DEBUG */
655
            break;
656
        default:
657
            dprintf("kvm_arch_handle_exit\n");
658
            ret = kvm_arch_handle_exit(env, run);
659
            break;
660
        }
661
    } while (ret > 0);
662

    
663
    if (env->exit_request) {
664
        env->exit_request = 0;
665
        env->exception_index = EXCP_INTERRUPT;
666
    }
667

    
668
    return ret;
669
}
670

    
671
void kvm_set_phys_mem(target_phys_addr_t start_addr,
672
                      ram_addr_t size,
673
                      ram_addr_t phys_offset)
674
{
675
    KVMState *s = kvm_state;
676
    ram_addr_t flags = phys_offset & ~TARGET_PAGE_MASK;
677
    KVMSlot *mem, old;
678
    int err;
679

    
680
    if (start_addr & ~TARGET_PAGE_MASK) {
681
        if (flags >= IO_MEM_UNASSIGNED) {
682
            if (!kvm_lookup_overlapping_slot(s, start_addr,
683
                                             start_addr + size)) {
684
                return;
685
            }
686
            fprintf(stderr, "Unaligned split of a KVM memory slot\n");
687
        } else {
688
            fprintf(stderr, "Only page-aligned memory slots supported\n");
689
        }
690
        abort();
691
    }
692

    
693
    /* KVM does not support read-only slots */
694
    phys_offset &= ~IO_MEM_ROM;
695

    
696
    while (1) {
697
        mem = kvm_lookup_overlapping_slot(s, start_addr, start_addr + size);
698
        if (!mem) {
699
            break;
700
        }
701

    
702
        if (flags < IO_MEM_UNASSIGNED && start_addr >= mem->start_addr &&
703
            (start_addr + size <= mem->start_addr + mem->memory_size) &&
704
            (phys_offset - start_addr == mem->phys_offset - mem->start_addr)) {
705
            /* The new slot fits into the existing one and comes with
706
             * identical parameters - nothing to be done. */
707
            return;
708
        }
709

    
710
        old = *mem;
711

    
712
        /* unregister the overlapping slot */
713
        mem->memory_size = 0;
714
        err = kvm_set_user_memory_region(s, mem);
715
        if (err) {
716
            fprintf(stderr, "%s: error unregistering overlapping slot: %s\n",
717
                    __func__, strerror(-err));
718
            abort();
719
        }
720

    
721
        /* Workaround for older KVM versions: we can't join slots, even not by
722
         * unregistering the previous ones and then registering the larger
723
         * slot. We have to maintain the existing fragmentation. Sigh.
724
         *
725
         * This workaround assumes that the new slot starts at the same
726
         * address as the first existing one. If not or if some overlapping
727
         * slot comes around later, we will fail (not seen in practice so far)
728
         * - and actually require a recent KVM version. */
729
        if (s->broken_set_mem_region &&
730
            old.start_addr == start_addr && old.memory_size < size &&
731
            flags < IO_MEM_UNASSIGNED) {
732
            mem = kvm_alloc_slot(s);
733
            mem->memory_size = old.memory_size;
734
            mem->start_addr = old.start_addr;
735
            mem->phys_offset = old.phys_offset;
736
            mem->flags = 0;
737

    
738
            err = kvm_set_user_memory_region(s, mem);
739
            if (err) {
740
                fprintf(stderr, "%s: error updating slot: %s\n", __func__,
741
                        strerror(-err));
742
                abort();
743
            }
744

    
745
            start_addr += old.memory_size;
746
            phys_offset += old.memory_size;
747
            size -= old.memory_size;
748
            continue;
749
        }
750

    
751
        /* register prefix slot */
752
        if (old.start_addr < start_addr) {
753
            mem = kvm_alloc_slot(s);
754
            mem->memory_size = start_addr - old.start_addr;
755
            mem->start_addr = old.start_addr;
756
            mem->phys_offset = old.phys_offset;
757
            mem->flags = 0;
758

    
759
            err = kvm_set_user_memory_region(s, mem);
760
            if (err) {
761
                fprintf(stderr, "%s: error registering prefix slot: %s\n",
762
                        __func__, strerror(-err));
763
                abort();
764
            }
765
        }
766

    
767
        /* register suffix slot */
768
        if (old.start_addr + old.memory_size > start_addr + size) {
769
            ram_addr_t size_delta;
770

    
771
            mem = kvm_alloc_slot(s);
772
            mem->start_addr = start_addr + size;
773
            size_delta = mem->start_addr - old.start_addr;
774
            mem->memory_size = old.memory_size - size_delta;
775
            mem->phys_offset = old.phys_offset + size_delta;
776
            mem->flags = 0;
777

    
778
            err = kvm_set_user_memory_region(s, mem);
779
            if (err) {
780
                fprintf(stderr, "%s: error registering suffix slot: %s\n",
781
                        __func__, strerror(-err));
782
                abort();
783
            }
784
        }
785
    }
786

    
787
    /* in case the KVM bug workaround already "consumed" the new slot */
788
    if (!size)
789
        return;
790

    
791
    /* KVM does not need to know about this memory */
792
    if (flags >= IO_MEM_UNASSIGNED)
793
        return;
794

    
795
    mem = kvm_alloc_slot(s);
796
    mem->memory_size = size;
797
    mem->start_addr = start_addr;
798
    mem->phys_offset = phys_offset;
799
    mem->flags = 0;
800

    
801
    err = kvm_set_user_memory_region(s, mem);
802
    if (err) {
803
        fprintf(stderr, "%s: error registering slot: %s\n", __func__,
804
                strerror(-err));
805
        abort();
806
    }
807
}
808

    
809
int kvm_ioctl(KVMState *s, int type, ...)
810
{
811
    int ret;
812
    void *arg;
813
    va_list ap;
814

    
815
    va_start(ap, type);
816
    arg = va_arg(ap, void *);
817
    va_end(ap);
818

    
819
    ret = ioctl(s->fd, type, arg);
820
    if (ret == -1)
821
        ret = -errno;
822

    
823
    return ret;
824
}
825

    
826
int kvm_vm_ioctl(KVMState *s, int type, ...)
827
{
828
    int ret;
829
    void *arg;
830
    va_list ap;
831

    
832
    va_start(ap, type);
833
    arg = va_arg(ap, void *);
834
    va_end(ap);
835

    
836
    ret = ioctl(s->vmfd, type, arg);
837
    if (ret == -1)
838
        ret = -errno;
839

    
840
    return ret;
841
}
842

    
843
int kvm_vcpu_ioctl(CPUState *env, int type, ...)
844
{
845
    int ret;
846
    void *arg;
847
    va_list ap;
848

    
849
    va_start(ap, type);
850
    arg = va_arg(ap, void *);
851
    va_end(ap);
852

    
853
    ret = ioctl(env->kvm_fd, type, arg);
854
    if (ret == -1)
855
        ret = -errno;
856

    
857
    return ret;
858
}
859

    
860
int kvm_has_sync_mmu(void)
861
{
862
#ifdef KVM_CAP_SYNC_MMU
863
    KVMState *s = kvm_state;
864

    
865
    return kvm_check_extension(s, KVM_CAP_SYNC_MMU);
866
#else
867
    return 0;
868
#endif
869
}
870

    
871
void kvm_setup_guest_memory(void *start, size_t size)
872
{
873
    if (!kvm_has_sync_mmu()) {
874
#ifdef MADV_DONTFORK
875
        int ret = madvise(start, size, MADV_DONTFORK);
876

    
877
        if (ret) {
878
            perror("madvice");
879
            exit(1);
880
        }
881
#else
882
        fprintf(stderr,
883
                "Need MADV_DONTFORK in absence of synchronous KVM MMU\n");
884
        exit(1);
885
#endif
886
    }
887
}
888

    
889
#ifdef KVM_CAP_SET_GUEST_DEBUG
890
static void on_vcpu(CPUState *env, void (*func)(void *data), void *data)
891
{
892
#ifdef CONFIG_IOTHREAD
893
    if (env == cpu_single_env) {
894
        func(data);
895
        return;
896
    }
897
    abort();
898
#else
899
    func(data);
900
#endif
901
}
902

    
903
struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *env,
904
                                                 target_ulong pc)
905
{
906
    struct kvm_sw_breakpoint *bp;
907

    
908
    QTAILQ_FOREACH(bp, &env->kvm_state->kvm_sw_breakpoints, entry) {
909
        if (bp->pc == pc)
910
            return bp;
911
    }
912
    return NULL;
913
}
914

    
915
int kvm_sw_breakpoints_active(CPUState *env)
916
{
917
    return !QTAILQ_EMPTY(&env->kvm_state->kvm_sw_breakpoints);
918
}
919

    
920
struct kvm_set_guest_debug_data {
921
    struct kvm_guest_debug dbg;
922
    CPUState *env;
923
    int err;
924
};
925

    
926
static void kvm_invoke_set_guest_debug(void *data)
927
{
928
    struct kvm_set_guest_debug_data *dbg_data = data;
929
    CPUState *env = dbg_data->env;
930

    
931
    if (env->kvm_state->regs_modified) {
932
        kvm_arch_put_registers(env);
933
        env->kvm_state->regs_modified = 0;
934
    }
935
    dbg_data->err = kvm_vcpu_ioctl(env, KVM_SET_GUEST_DEBUG, &dbg_data->dbg);
936
}
937

    
938
int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
939
{
940
    struct kvm_set_guest_debug_data data;
941

    
942
    data.dbg.control = 0;
943
    if (env->singlestep_enabled)
944
        data.dbg.control = KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP;
945

    
946
    kvm_arch_update_guest_debug(env, &data.dbg);
947
    data.dbg.control |= reinject_trap;
948
    data.env = env;
949

    
950
    on_vcpu(env, kvm_invoke_set_guest_debug, &data);
951
    return data.err;
952
}
953

    
954
int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
955
                          target_ulong len, int type)
956
{
957
    struct kvm_sw_breakpoint *bp;
958
    CPUState *env;
959
    int err;
960

    
961
    if (type == GDB_BREAKPOINT_SW) {
962
        bp = kvm_find_sw_breakpoint(current_env, addr);
963
        if (bp) {
964
            bp->use_count++;
965
            return 0;
966
        }
967

    
968
        bp = qemu_malloc(sizeof(struct kvm_sw_breakpoint));
969
        if (!bp)
970
            return -ENOMEM;
971

    
972
        bp->pc = addr;
973
        bp->use_count = 1;
974
        err = kvm_arch_insert_sw_breakpoint(current_env, bp);
975
        if (err) {
976
            free(bp);
977
            return err;
978
        }
979

    
980
        QTAILQ_INSERT_HEAD(&current_env->kvm_state->kvm_sw_breakpoints,
981
                          bp, entry);
982
    } else {
983
        err = kvm_arch_insert_hw_breakpoint(addr, len, type);
984
        if (err)
985
            return err;
986
    }
987

    
988
    for (env = first_cpu; env != NULL; env = env->next_cpu) {
989
        err = kvm_update_guest_debug(env, 0);
990
        if (err)
991
            return err;
992
    }
993
    return 0;
994
}
995

    
996
int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
997
                          target_ulong len, int type)
998
{
999
    struct kvm_sw_breakpoint *bp;
1000
    CPUState *env;
1001
    int err;
1002

    
1003
    if (type == GDB_BREAKPOINT_SW) {
1004
        bp = kvm_find_sw_breakpoint(current_env, addr);
1005
        if (!bp)
1006
            return -ENOENT;
1007

    
1008
        if (bp->use_count > 1) {
1009
            bp->use_count--;
1010
            return 0;
1011
        }
1012

    
1013
        err = kvm_arch_remove_sw_breakpoint(current_env, bp);
1014
        if (err)
1015
            return err;
1016

    
1017
        QTAILQ_REMOVE(&current_env->kvm_state->kvm_sw_breakpoints, bp, entry);
1018
        qemu_free(bp);
1019
    } else {
1020
        err = kvm_arch_remove_hw_breakpoint(addr, len, type);
1021
        if (err)
1022
            return err;
1023
    }
1024

    
1025
    for (env = first_cpu; env != NULL; env = env->next_cpu) {
1026
        err = kvm_update_guest_debug(env, 0);
1027
        if (err)
1028
            return err;
1029
    }
1030
    return 0;
1031
}
1032

    
1033
void kvm_remove_all_breakpoints(CPUState *current_env)
1034
{
1035
    struct kvm_sw_breakpoint *bp, *next;
1036
    KVMState *s = current_env->kvm_state;
1037
    CPUState *env;
1038

    
1039
    QTAILQ_FOREACH_SAFE(bp, &s->kvm_sw_breakpoints, entry, next) {
1040
        if (kvm_arch_remove_sw_breakpoint(current_env, bp) != 0) {
1041
            /* Try harder to find a CPU that currently sees the breakpoint. */
1042
            for (env = first_cpu; env != NULL; env = env->next_cpu) {
1043
                if (kvm_arch_remove_sw_breakpoint(env, bp) == 0)
1044
                    break;
1045
            }
1046
        }
1047
    }
1048
    kvm_arch_remove_all_hw_breakpoints();
1049

    
1050
    for (env = first_cpu; env != NULL; env = env->next_cpu)
1051
        kvm_update_guest_debug(env, 0);
1052
}
1053

    
1054
#else /* !KVM_CAP_SET_GUEST_DEBUG */
1055

    
1056
int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
1057
{
1058
    return -EINVAL;
1059
}
1060

    
1061
int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
1062
                          target_ulong len, int type)
1063
{
1064
    return -EINVAL;
1065
}
1066

    
1067
int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
1068
                          target_ulong len, int type)
1069
{
1070
    return -EINVAL;
1071
}
1072

    
1073
void kvm_remove_all_breakpoints(CPUState *current_env)
1074
{
1075
}
1076
#endif /* !KVM_CAP_SET_GUEST_DEBUG */