Statistics
| Branch: | Revision:

root / kvm-all.c @ 62a2744c

History | View | Annotate | Download (27.6 kB)

1
/*
2
 * QEMU KVM support
3
 *
4
 * Copyright IBM, Corp. 2008
5
 *           Red Hat, Inc. 2008
6
 *
7
 * Authors:
8
 *  Anthony Liguori   <aliguori@us.ibm.com>
9
 *  Glauber Costa     <gcosta@redhat.com>
10
 *
11
 * This work is licensed under the terms of the GNU GPL, version 2 or later.
12
 * See the COPYING file in the top-level directory.
13
 *
14
 */
15

    
16
#include <sys/types.h>
17
#include <sys/ioctl.h>
18
#include <sys/mman.h>
19
#include <stdarg.h>
20

    
21
#include <linux/kvm.h>
22

    
23
#include "qemu-common.h"
24
#include "sysemu.h"
25
#include "hw/hw.h"
26
#include "gdbstub.h"
27
#include "kvm.h"
28

    
29
/* KVM uses PAGE_SIZE in it's definition of COALESCED_MMIO_MAX */
30
#define PAGE_SIZE TARGET_PAGE_SIZE
31

    
32
//#define DEBUG_KVM
33

    
34
#ifdef DEBUG_KVM
35
#define dprintf(fmt, ...) \
36
    do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
37
#else
38
#define dprintf(fmt, ...) \
39
    do { } while (0)
40
#endif
41

    
42
typedef struct KVMSlot
43
{
44
    target_phys_addr_t start_addr;
45
    ram_addr_t memory_size;
46
    ram_addr_t phys_offset;
47
    int slot;
48
    int flags;
49
} KVMSlot;
50

    
51
typedef struct kvm_dirty_log KVMDirtyLog;
52

    
53
int kvm_allowed = 0;
54

    
55
struct KVMState
56
{
57
    KVMSlot slots[32];
58
    int fd;
59
    int vmfd;
60
    int regs_modified;
61
    int coalesced_mmio;
62
#ifdef KVM_CAP_COALESCED_MMIO
63
    struct kvm_coalesced_mmio_ring *coalesced_mmio_ring;
64
#endif
65
    int broken_set_mem_region;
66
    int migration_log;
67
    int vcpu_events;
68
#ifdef KVM_CAP_SET_GUEST_DEBUG
69
    struct kvm_sw_breakpoint_head kvm_sw_breakpoints;
70
#endif
71
    int irqchip_in_kernel;
72
    int pit_in_kernel;
73
};
74

    
75
static KVMState *kvm_state;
76

    
77
static KVMSlot *kvm_alloc_slot(KVMState *s)
78
{
79
    int i;
80

    
81
    for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
82
        /* KVM private memory slots */
83
        if (i >= 8 && i < 12)
84
            continue;
85
        if (s->slots[i].memory_size == 0)
86
            return &s->slots[i];
87
    }
88

    
89
    fprintf(stderr, "%s: no free slot available\n", __func__);
90
    abort();
91
}
92

    
93
static KVMSlot *kvm_lookup_matching_slot(KVMState *s,
94
                                         target_phys_addr_t start_addr,
95
                                         target_phys_addr_t end_addr)
96
{
97
    int i;
98

    
99
    for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
100
        KVMSlot *mem = &s->slots[i];
101

    
102
        if (start_addr == mem->start_addr &&
103
            end_addr == mem->start_addr + mem->memory_size) {
104
            return mem;
105
        }
106
    }
107

    
108
    return NULL;
109
}
110

    
111
/*
112
 * Find overlapping slot with lowest start address
113
 */
114
static KVMSlot *kvm_lookup_overlapping_slot(KVMState *s,
115
                                            target_phys_addr_t start_addr,
116
                                            target_phys_addr_t end_addr)
117
{
118
    KVMSlot *found = NULL;
119
    int i;
120

    
121
    for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
122
        KVMSlot *mem = &s->slots[i];
123

    
124
        if (mem->memory_size == 0 ||
125
            (found && found->start_addr < mem->start_addr)) {
126
            continue;
127
        }
128

    
129
        if (end_addr > mem->start_addr &&
130
            start_addr < mem->start_addr + mem->memory_size) {
131
            found = mem;
132
        }
133
    }
134

    
135
    return found;
136
}
137

    
138
static int kvm_set_user_memory_region(KVMState *s, KVMSlot *slot)
139
{
140
    struct kvm_userspace_memory_region mem;
141

    
142
    mem.slot = slot->slot;
143
    mem.guest_phys_addr = slot->start_addr;
144
    mem.memory_size = slot->memory_size;
145
    mem.userspace_addr = (unsigned long)qemu_get_ram_ptr(slot->phys_offset);
146
    mem.flags = slot->flags;
147
    if (s->migration_log) {
148
        mem.flags |= KVM_MEM_LOG_DIRTY_PAGES;
149
    }
150
    return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
151
}
152

    
153
static void kvm_reset_vcpu(void *opaque)
154
{
155
    CPUState *env = opaque;
156

    
157
    kvm_arch_reset_vcpu(env);
158
    if (kvm_arch_put_registers(env)) {
159
        fprintf(stderr, "Fatal: kvm vcpu reset failed\n");
160
        abort();
161
    }
162
}
163

    
164
int kvm_irqchip_in_kernel(void)
165
{
166
    return kvm_state->irqchip_in_kernel;
167
}
168

    
169
int kvm_pit_in_kernel(void)
170
{
171
    return kvm_state->pit_in_kernel;
172
}
173

    
174

    
175
int kvm_init_vcpu(CPUState *env)
176
{
177
    KVMState *s = kvm_state;
178
    long mmap_size;
179
    int ret;
180

    
181
    dprintf("kvm_init_vcpu\n");
182

    
183
    ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, env->cpu_index);
184
    if (ret < 0) {
185
        dprintf("kvm_create_vcpu failed\n");
186
        goto err;
187
    }
188

    
189
    env->kvm_fd = ret;
190
    env->kvm_state = s;
191

    
192
    mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
193
    if (mmap_size < 0) {
194
        dprintf("KVM_GET_VCPU_MMAP_SIZE failed\n");
195
        goto err;
196
    }
197

    
198
    env->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
199
                        env->kvm_fd, 0);
200
    if (env->kvm_run == MAP_FAILED) {
201
        ret = -errno;
202
        dprintf("mmap'ing vcpu state failed\n");
203
        goto err;
204
    }
205

    
206
#ifdef KVM_CAP_COALESCED_MMIO
207
    if (s->coalesced_mmio && !s->coalesced_mmio_ring)
208
        s->coalesced_mmio_ring = (void *) env->kvm_run +
209
                s->coalesced_mmio * PAGE_SIZE;
210
#endif
211

    
212
    ret = kvm_arch_init_vcpu(env);
213
    if (ret == 0) {
214
        qemu_register_reset(kvm_reset_vcpu, env);
215
        kvm_arch_reset_vcpu(env);
216
        ret = kvm_arch_put_registers(env);
217
    }
218
err:
219
    return ret;
220
}
221

    
222
/*
223
 * dirty pages logging control
224
 */
225
static int kvm_dirty_pages_log_change(target_phys_addr_t phys_addr,
226
                                      ram_addr_t size, int flags, int mask)
227
{
228
    KVMState *s = kvm_state;
229
    KVMSlot *mem = kvm_lookup_matching_slot(s, phys_addr, phys_addr + size);
230
    int old_flags;
231

    
232
    if (mem == NULL)  {
233
            fprintf(stderr, "BUG: %s: invalid parameters " TARGET_FMT_plx "-"
234
                    TARGET_FMT_plx "\n", __func__, phys_addr,
235
                    (target_phys_addr_t)(phys_addr + size - 1));
236
            return -EINVAL;
237
    }
238

    
239
    old_flags = mem->flags;
240

    
241
    flags = (mem->flags & ~mask) | flags;
242
    mem->flags = flags;
243

    
244
    /* If nothing changed effectively, no need to issue ioctl */
245
    if (s->migration_log) {
246
        flags |= KVM_MEM_LOG_DIRTY_PAGES;
247
    }
248
    if (flags == old_flags) {
249
            return 0;
250
    }
251

    
252
    return kvm_set_user_memory_region(s, mem);
253
}
254

    
255
int kvm_log_start(target_phys_addr_t phys_addr, ram_addr_t size)
256
{
257
        return kvm_dirty_pages_log_change(phys_addr, size,
258
                                          KVM_MEM_LOG_DIRTY_PAGES,
259
                                          KVM_MEM_LOG_DIRTY_PAGES);
260
}
261

    
262
int kvm_log_stop(target_phys_addr_t phys_addr, ram_addr_t size)
263
{
264
        return kvm_dirty_pages_log_change(phys_addr, size,
265
                                          0,
266
                                          KVM_MEM_LOG_DIRTY_PAGES);
267
}
268

    
269
int kvm_set_migration_log(int enable)
270
{
271
    KVMState *s = kvm_state;
272
    KVMSlot *mem;
273
    int i, err;
274

    
275
    s->migration_log = enable;
276

    
277
    for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
278
        mem = &s->slots[i];
279

    
280
        if (!!(mem->flags & KVM_MEM_LOG_DIRTY_PAGES) == enable) {
281
            continue;
282
        }
283
        err = kvm_set_user_memory_region(s, mem);
284
        if (err) {
285
            return err;
286
        }
287
    }
288
    return 0;
289
}
290

    
291
static int test_le_bit(unsigned long nr, unsigned char *addr)
292
{
293
    return (addr[nr >> 3] >> (nr & 7)) & 1;
294
}
295

    
296
/**
297
 * kvm_physical_sync_dirty_bitmap - Grab dirty bitmap from kernel space
298
 * This function updates qemu's dirty bitmap using cpu_physical_memory_set_dirty().
299
 * This means all bits are set to dirty.
300
 *
301
 * @start_add: start of logged region.
302
 * @end_addr: end of logged region.
303
 */
304
int kvm_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,
305
                                   target_phys_addr_t end_addr)
306
{
307
    KVMState *s = kvm_state;
308
    unsigned long size, allocated_size = 0;
309
    target_phys_addr_t phys_addr;
310
    ram_addr_t addr;
311
    KVMDirtyLog d;
312
    KVMSlot *mem;
313
    int ret = 0;
314

    
315
    d.dirty_bitmap = NULL;
316
    while (start_addr < end_addr) {
317
        mem = kvm_lookup_overlapping_slot(s, start_addr, end_addr);
318
        if (mem == NULL) {
319
            break;
320
        }
321

    
322
        size = ((mem->memory_size >> TARGET_PAGE_BITS) + 7) / 8;
323
        if (!d.dirty_bitmap) {
324
            d.dirty_bitmap = qemu_malloc(size);
325
        } else if (size > allocated_size) {
326
            d.dirty_bitmap = qemu_realloc(d.dirty_bitmap, size);
327
        }
328
        allocated_size = size;
329
        memset(d.dirty_bitmap, 0, allocated_size);
330

    
331
        d.slot = mem->slot;
332

    
333
        if (kvm_vm_ioctl(s, KVM_GET_DIRTY_LOG, &d) == -1) {
334
            dprintf("ioctl failed %d\n", errno);
335
            ret = -1;
336
            break;
337
        }
338

    
339
        for (phys_addr = mem->start_addr, addr = mem->phys_offset;
340
             phys_addr < mem->start_addr + mem->memory_size;
341
             phys_addr += TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
342
            unsigned char *bitmap = (unsigned char *)d.dirty_bitmap;
343
            unsigned nr = (phys_addr - mem->start_addr) >> TARGET_PAGE_BITS;
344

    
345
            if (test_le_bit(nr, bitmap)) {
346
                cpu_physical_memory_set_dirty(addr);
347
            }
348
        }
349
        start_addr = phys_addr;
350
    }
351
    qemu_free(d.dirty_bitmap);
352

    
353
    return ret;
354
}
355

    
356
int kvm_coalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
357
{
358
    int ret = -ENOSYS;
359
#ifdef KVM_CAP_COALESCED_MMIO
360
    KVMState *s = kvm_state;
361

    
362
    if (s->coalesced_mmio) {
363
        struct kvm_coalesced_mmio_zone zone;
364

    
365
        zone.addr = start;
366
        zone.size = size;
367

    
368
        ret = kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
369
    }
370
#endif
371

    
372
    return ret;
373
}
374

    
375
int kvm_uncoalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
376
{
377
    int ret = -ENOSYS;
378
#ifdef KVM_CAP_COALESCED_MMIO
379
    KVMState *s = kvm_state;
380

    
381
    if (s->coalesced_mmio) {
382
        struct kvm_coalesced_mmio_zone zone;
383

    
384
        zone.addr = start;
385
        zone.size = size;
386

    
387
        ret = kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
388
    }
389
#endif
390

    
391
    return ret;
392
}
393

    
394
int kvm_check_extension(KVMState *s, unsigned int extension)
395
{
396
    int ret;
397

    
398
    ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, extension);
399
    if (ret < 0) {
400
        ret = 0;
401
    }
402

    
403
    return ret;
404
}
405

    
406
int kvm_init(int smp_cpus)
407
{
408
    static const char upgrade_note[] =
409
        "Please upgrade to at least kernel 2.6.29 or recent kvm-kmod\n"
410
        "(see http://sourceforge.net/projects/kvm).\n";
411
    KVMState *s;
412
    int ret;
413
    int i;
414

    
415
    if (smp_cpus > 1) {
416
        fprintf(stderr, "No SMP KVM support, use '-smp 1'\n");
417
        return -EINVAL;
418
    }
419

    
420
    s = qemu_mallocz(sizeof(KVMState));
421

    
422
#ifdef KVM_CAP_SET_GUEST_DEBUG
423
    QTAILQ_INIT(&s->kvm_sw_breakpoints);
424
#endif
425
    for (i = 0; i < ARRAY_SIZE(s->slots); i++)
426
        s->slots[i].slot = i;
427

    
428
    s->vmfd = -1;
429
    s->fd = qemu_open("/dev/kvm", O_RDWR);
430
    if (s->fd == -1) {
431
        fprintf(stderr, "Could not access KVM kernel module: %m\n");
432
        ret = -errno;
433
        goto err;
434
    }
435

    
436
    ret = kvm_ioctl(s, KVM_GET_API_VERSION, 0);
437
    if (ret < KVM_API_VERSION) {
438
        if (ret > 0)
439
            ret = -EINVAL;
440
        fprintf(stderr, "kvm version too old\n");
441
        goto err;
442
    }
443

    
444
    if (ret > KVM_API_VERSION) {
445
        ret = -EINVAL;
446
        fprintf(stderr, "kvm version not supported\n");
447
        goto err;
448
    }
449

    
450
    s->vmfd = kvm_ioctl(s, KVM_CREATE_VM, 0);
451
    if (s->vmfd < 0)
452
        goto err;
453

    
454
    /* initially, KVM allocated its own memory and we had to jump through
455
     * hooks to make phys_ram_base point to this.  Modern versions of KVM
456
     * just use a user allocated buffer so we can use regular pages
457
     * unmodified.  Make sure we have a sufficiently modern version of KVM.
458
     */
459
    if (!kvm_check_extension(s, KVM_CAP_USER_MEMORY)) {
460
        ret = -EINVAL;
461
        fprintf(stderr, "kvm does not support KVM_CAP_USER_MEMORY\n%s",
462
                upgrade_note);
463
        goto err;
464
    }
465

    
466
    /* There was a nasty bug in < kvm-80 that prevents memory slots from being
467
     * destroyed properly.  Since we rely on this capability, refuse to work
468
     * with any kernel without this capability. */
469
    if (!kvm_check_extension(s, KVM_CAP_DESTROY_MEMORY_REGION_WORKS)) {
470
        ret = -EINVAL;
471

    
472
        fprintf(stderr,
473
                "KVM kernel module broken (DESTROY_MEMORY_REGION).\n%s",
474
                upgrade_note);
475
        goto err;
476
    }
477

    
478
    s->coalesced_mmio = 0;
479
#ifdef KVM_CAP_COALESCED_MMIO
480
    s->coalesced_mmio = kvm_check_extension(s, KVM_CAP_COALESCED_MMIO);
481
    s->coalesced_mmio_ring = NULL;
482
#endif
483

    
484
    s->broken_set_mem_region = 1;
485
#ifdef KVM_CAP_JOIN_MEMORY_REGIONS_WORKS
486
    ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, KVM_CAP_JOIN_MEMORY_REGIONS_WORKS);
487
    if (ret > 0) {
488
        s->broken_set_mem_region = 0;
489
    }
490
#endif
491

    
492
    s->vcpu_events = 0;
493
#ifdef KVM_CAP_VCPU_EVENTS
494
    s->vcpu_events = kvm_check_extension(s, KVM_CAP_VCPU_EVENTS);
495
#endif
496

    
497
    ret = kvm_arch_init(s, smp_cpus);
498
    if (ret < 0)
499
        goto err;
500

    
501
    kvm_state = s;
502

    
503
    return 0;
504

    
505
err:
506
    if (s) {
507
        if (s->vmfd != -1)
508
            close(s->vmfd);
509
        if (s->fd != -1)
510
            close(s->fd);
511
    }
512
    qemu_free(s);
513

    
514
    return ret;
515
}
516

    
517
static int kvm_handle_io(uint16_t port, void *data, int direction, int size,
518
                         uint32_t count)
519
{
520
    int i;
521
    uint8_t *ptr = data;
522

    
523
    for (i = 0; i < count; i++) {
524
        if (direction == KVM_EXIT_IO_IN) {
525
            switch (size) {
526
            case 1:
527
                stb_p(ptr, cpu_inb(port));
528
                break;
529
            case 2:
530
                stw_p(ptr, cpu_inw(port));
531
                break;
532
            case 4:
533
                stl_p(ptr, cpu_inl(port));
534
                break;
535
            }
536
        } else {
537
            switch (size) {
538
            case 1:
539
                cpu_outb(port, ldub_p(ptr));
540
                break;
541
            case 2:
542
                cpu_outw(port, lduw_p(ptr));
543
                break;
544
            case 4:
545
                cpu_outl(port, ldl_p(ptr));
546
                break;
547
            }
548
        }
549

    
550
        ptr += size;
551
    }
552

    
553
    return 1;
554
}
555

    
556
void kvm_flush_coalesced_mmio_buffer(void)
557
{
558
#ifdef KVM_CAP_COALESCED_MMIO
559
    KVMState *s = kvm_state;
560
    if (s->coalesced_mmio_ring) {
561
        struct kvm_coalesced_mmio_ring *ring = s->coalesced_mmio_ring;
562
        while (ring->first != ring->last) {
563
            struct kvm_coalesced_mmio *ent;
564

    
565
            ent = &ring->coalesced_mmio[ring->first];
566

    
567
            cpu_physical_memory_write(ent->phys_addr, ent->data, ent->len);
568
            /* FIXME smp_wmb() */
569
            ring->first = (ring->first + 1) % KVM_COALESCED_MMIO_MAX;
570
        }
571
    }
572
#endif
573
}
574

    
575
void kvm_cpu_synchronize_state(CPUState *env)
576
{
577
    if (!env->kvm_state->regs_modified) {
578
        kvm_arch_get_registers(env);
579
        env->kvm_state->regs_modified = 1;
580
    }
581
}
582

    
583
int kvm_cpu_exec(CPUState *env)
584
{
585
    struct kvm_run *run = env->kvm_run;
586
    int ret;
587

    
588
    dprintf("kvm_cpu_exec()\n");
589

    
590
    do {
591
        if (env->exit_request) {
592
            dprintf("interrupt exit requested\n");
593
            ret = 0;
594
            break;
595
        }
596

    
597
        if (env->kvm_state->regs_modified) {
598
            kvm_arch_put_registers(env);
599
            env->kvm_state->regs_modified = 0;
600
        }
601

    
602
        kvm_arch_pre_run(env, run);
603
        qemu_mutex_unlock_iothread();
604
        ret = kvm_vcpu_ioctl(env, KVM_RUN, 0);
605
        qemu_mutex_lock_iothread();
606
        kvm_arch_post_run(env, run);
607

    
608
        if (ret == -EINTR || ret == -EAGAIN) {
609
            dprintf("io window exit\n");
610
            ret = 0;
611
            break;
612
        }
613

    
614
        if (ret < 0) {
615
            dprintf("kvm run failed %s\n", strerror(-ret));
616
            abort();
617
        }
618

    
619
        kvm_flush_coalesced_mmio_buffer();
620

    
621
        ret = 0; /* exit loop */
622
        switch (run->exit_reason) {
623
        case KVM_EXIT_IO:
624
            dprintf("handle_io\n");
625
            ret = kvm_handle_io(run->io.port,
626
                                (uint8_t *)run + run->io.data_offset,
627
                                run->io.direction,
628
                                run->io.size,
629
                                run->io.count);
630
            break;
631
        case KVM_EXIT_MMIO:
632
            dprintf("handle_mmio\n");
633
            cpu_physical_memory_rw(run->mmio.phys_addr,
634
                                   run->mmio.data,
635
                                   run->mmio.len,
636
                                   run->mmio.is_write);
637
            ret = 1;
638
            break;
639
        case KVM_EXIT_IRQ_WINDOW_OPEN:
640
            dprintf("irq_window_open\n");
641
            break;
642
        case KVM_EXIT_SHUTDOWN:
643
            dprintf("shutdown\n");
644
            qemu_system_reset_request();
645
            ret = 1;
646
            break;
647
        case KVM_EXIT_UNKNOWN:
648
            dprintf("kvm_exit_unknown\n");
649
            break;
650
        case KVM_EXIT_FAIL_ENTRY:
651
            dprintf("kvm_exit_fail_entry\n");
652
            break;
653
        case KVM_EXIT_EXCEPTION:
654
            dprintf("kvm_exit_exception\n");
655
            break;
656
        case KVM_EXIT_DEBUG:
657
            dprintf("kvm_exit_debug\n");
658
#ifdef KVM_CAP_SET_GUEST_DEBUG
659
            if (kvm_arch_debug(&run->debug.arch)) {
660
                gdb_set_stop_cpu(env);
661
                vm_stop(EXCP_DEBUG);
662
                env->exception_index = EXCP_DEBUG;
663
                return 0;
664
            }
665
            /* re-enter, this exception was guest-internal */
666
            ret = 1;
667
#endif /* KVM_CAP_SET_GUEST_DEBUG */
668
            break;
669
        default:
670
            dprintf("kvm_arch_handle_exit\n");
671
            ret = kvm_arch_handle_exit(env, run);
672
            break;
673
        }
674
    } while (ret > 0);
675

    
676
    if (env->exit_request) {
677
        env->exit_request = 0;
678
        env->exception_index = EXCP_INTERRUPT;
679
    }
680

    
681
    return ret;
682
}
683

    
684
void kvm_set_phys_mem(target_phys_addr_t start_addr,
685
                      ram_addr_t size,
686
                      ram_addr_t phys_offset)
687
{
688
    KVMState *s = kvm_state;
689
    ram_addr_t flags = phys_offset & ~TARGET_PAGE_MASK;
690
    KVMSlot *mem, old;
691
    int err;
692

    
693
    if (start_addr & ~TARGET_PAGE_MASK) {
694
        if (flags >= IO_MEM_UNASSIGNED) {
695
            if (!kvm_lookup_overlapping_slot(s, start_addr,
696
                                             start_addr + size)) {
697
                return;
698
            }
699
            fprintf(stderr, "Unaligned split of a KVM memory slot\n");
700
        } else {
701
            fprintf(stderr, "Only page-aligned memory slots supported\n");
702
        }
703
        abort();
704
    }
705

    
706
    /* KVM does not support read-only slots */
707
    phys_offset &= ~IO_MEM_ROM;
708

    
709
    while (1) {
710
        mem = kvm_lookup_overlapping_slot(s, start_addr, start_addr + size);
711
        if (!mem) {
712
            break;
713
        }
714

    
715
        if (flags < IO_MEM_UNASSIGNED && start_addr >= mem->start_addr &&
716
            (start_addr + size <= mem->start_addr + mem->memory_size) &&
717
            (phys_offset - start_addr == mem->phys_offset - mem->start_addr)) {
718
            /* The new slot fits into the existing one and comes with
719
             * identical parameters - nothing to be done. */
720
            return;
721
        }
722

    
723
        old = *mem;
724

    
725
        /* unregister the overlapping slot */
726
        mem->memory_size = 0;
727
        err = kvm_set_user_memory_region(s, mem);
728
        if (err) {
729
            fprintf(stderr, "%s: error unregistering overlapping slot: %s\n",
730
                    __func__, strerror(-err));
731
            abort();
732
        }
733

    
734
        /* Workaround for older KVM versions: we can't join slots, even not by
735
         * unregistering the previous ones and then registering the larger
736
         * slot. We have to maintain the existing fragmentation. Sigh.
737
         *
738
         * This workaround assumes that the new slot starts at the same
739
         * address as the first existing one. If not or if some overlapping
740
         * slot comes around later, we will fail (not seen in practice so far)
741
         * - and actually require a recent KVM version. */
742
        if (s->broken_set_mem_region &&
743
            old.start_addr == start_addr && old.memory_size < size &&
744
            flags < IO_MEM_UNASSIGNED) {
745
            mem = kvm_alloc_slot(s);
746
            mem->memory_size = old.memory_size;
747
            mem->start_addr = old.start_addr;
748
            mem->phys_offset = old.phys_offset;
749
            mem->flags = 0;
750

    
751
            err = kvm_set_user_memory_region(s, mem);
752
            if (err) {
753
                fprintf(stderr, "%s: error updating slot: %s\n", __func__,
754
                        strerror(-err));
755
                abort();
756
            }
757

    
758
            start_addr += old.memory_size;
759
            phys_offset += old.memory_size;
760
            size -= old.memory_size;
761
            continue;
762
        }
763

    
764
        /* register prefix slot */
765
        if (old.start_addr < start_addr) {
766
            mem = kvm_alloc_slot(s);
767
            mem->memory_size = start_addr - old.start_addr;
768
            mem->start_addr = old.start_addr;
769
            mem->phys_offset = old.phys_offset;
770
            mem->flags = 0;
771

    
772
            err = kvm_set_user_memory_region(s, mem);
773
            if (err) {
774
                fprintf(stderr, "%s: error registering prefix slot: %s\n",
775
                        __func__, strerror(-err));
776
                abort();
777
            }
778
        }
779

    
780
        /* register suffix slot */
781
        if (old.start_addr + old.memory_size > start_addr + size) {
782
            ram_addr_t size_delta;
783

    
784
            mem = kvm_alloc_slot(s);
785
            mem->start_addr = start_addr + size;
786
            size_delta = mem->start_addr - old.start_addr;
787
            mem->memory_size = old.memory_size - size_delta;
788
            mem->phys_offset = old.phys_offset + size_delta;
789
            mem->flags = 0;
790

    
791
            err = kvm_set_user_memory_region(s, mem);
792
            if (err) {
793
                fprintf(stderr, "%s: error registering suffix slot: %s\n",
794
                        __func__, strerror(-err));
795
                abort();
796
            }
797
        }
798
    }
799

    
800
    /* in case the KVM bug workaround already "consumed" the new slot */
801
    if (!size)
802
        return;
803

    
804
    /* KVM does not need to know about this memory */
805
    if (flags >= IO_MEM_UNASSIGNED)
806
        return;
807

    
808
    mem = kvm_alloc_slot(s);
809
    mem->memory_size = size;
810
    mem->start_addr = start_addr;
811
    mem->phys_offset = phys_offset;
812
    mem->flags = 0;
813

    
814
    err = kvm_set_user_memory_region(s, mem);
815
    if (err) {
816
        fprintf(stderr, "%s: error registering slot: %s\n", __func__,
817
                strerror(-err));
818
        abort();
819
    }
820
}
821

    
822
int kvm_ioctl(KVMState *s, int type, ...)
823
{
824
    int ret;
825
    void *arg;
826
    va_list ap;
827

    
828
    va_start(ap, type);
829
    arg = va_arg(ap, void *);
830
    va_end(ap);
831

    
832
    ret = ioctl(s->fd, type, arg);
833
    if (ret == -1)
834
        ret = -errno;
835

    
836
    return ret;
837
}
838

    
839
int kvm_vm_ioctl(KVMState *s, int type, ...)
840
{
841
    int ret;
842
    void *arg;
843
    va_list ap;
844

    
845
    va_start(ap, type);
846
    arg = va_arg(ap, void *);
847
    va_end(ap);
848

    
849
    ret = ioctl(s->vmfd, type, arg);
850
    if (ret == -1)
851
        ret = -errno;
852

    
853
    return ret;
854
}
855

    
856
int kvm_vcpu_ioctl(CPUState *env, int type, ...)
857
{
858
    int ret;
859
    void *arg;
860
    va_list ap;
861

    
862
    va_start(ap, type);
863
    arg = va_arg(ap, void *);
864
    va_end(ap);
865

    
866
    ret = ioctl(env->kvm_fd, type, arg);
867
    if (ret == -1)
868
        ret = -errno;
869

    
870
    return ret;
871
}
872

    
873
int kvm_has_sync_mmu(void)
874
{
875
#ifdef KVM_CAP_SYNC_MMU
876
    KVMState *s = kvm_state;
877

    
878
    return kvm_check_extension(s, KVM_CAP_SYNC_MMU);
879
#else
880
    return 0;
881
#endif
882
}
883

    
884
int kvm_has_vcpu_events(void)
885
{
886
    return kvm_state->vcpu_events;
887
}
888

    
889
void kvm_setup_guest_memory(void *start, size_t size)
890
{
891
    if (!kvm_has_sync_mmu()) {
892
#ifdef MADV_DONTFORK
893
        int ret = madvise(start, size, MADV_DONTFORK);
894

    
895
        if (ret) {
896
            perror("madvice");
897
            exit(1);
898
        }
899
#else
900
        fprintf(stderr,
901
                "Need MADV_DONTFORK in absence of synchronous KVM MMU\n");
902
        exit(1);
903
#endif
904
    }
905
}
906

    
907
#ifdef KVM_CAP_SET_GUEST_DEBUG
908
static void on_vcpu(CPUState *env, void (*func)(void *data), void *data)
909
{
910
#ifdef CONFIG_IOTHREAD
911
    if (env == cpu_single_env) {
912
        func(data);
913
        return;
914
    }
915
    abort();
916
#else
917
    func(data);
918
#endif
919
}
920

    
921
struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *env,
922
                                                 target_ulong pc)
923
{
924
    struct kvm_sw_breakpoint *bp;
925

    
926
    QTAILQ_FOREACH(bp, &env->kvm_state->kvm_sw_breakpoints, entry) {
927
        if (bp->pc == pc)
928
            return bp;
929
    }
930
    return NULL;
931
}
932

    
933
int kvm_sw_breakpoints_active(CPUState *env)
934
{
935
    return !QTAILQ_EMPTY(&env->kvm_state->kvm_sw_breakpoints);
936
}
937

    
938
struct kvm_set_guest_debug_data {
939
    struct kvm_guest_debug dbg;
940
    CPUState *env;
941
    int err;
942
};
943

    
944
static void kvm_invoke_set_guest_debug(void *data)
945
{
946
    struct kvm_set_guest_debug_data *dbg_data = data;
947
    CPUState *env = dbg_data->env;
948

    
949
    if (env->kvm_state->regs_modified) {
950
        kvm_arch_put_registers(env);
951
        env->kvm_state->regs_modified = 0;
952
    }
953
    dbg_data->err = kvm_vcpu_ioctl(env, KVM_SET_GUEST_DEBUG, &dbg_data->dbg);
954
}
955

    
956
int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
957
{
958
    struct kvm_set_guest_debug_data data;
959

    
960
    data.dbg.control = 0;
961
    if (env->singlestep_enabled)
962
        data.dbg.control = KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP;
963

    
964
    kvm_arch_update_guest_debug(env, &data.dbg);
965
    data.dbg.control |= reinject_trap;
966
    data.env = env;
967

    
968
    on_vcpu(env, kvm_invoke_set_guest_debug, &data);
969
    return data.err;
970
}
971

    
972
int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
973
                          target_ulong len, int type)
974
{
975
    struct kvm_sw_breakpoint *bp;
976
    CPUState *env;
977
    int err;
978

    
979
    if (type == GDB_BREAKPOINT_SW) {
980
        bp = kvm_find_sw_breakpoint(current_env, addr);
981
        if (bp) {
982
            bp->use_count++;
983
            return 0;
984
        }
985

    
986
        bp = qemu_malloc(sizeof(struct kvm_sw_breakpoint));
987
        if (!bp)
988
            return -ENOMEM;
989

    
990
        bp->pc = addr;
991
        bp->use_count = 1;
992
        err = kvm_arch_insert_sw_breakpoint(current_env, bp);
993
        if (err) {
994
            free(bp);
995
            return err;
996
        }
997

    
998
        QTAILQ_INSERT_HEAD(&current_env->kvm_state->kvm_sw_breakpoints,
999
                          bp, entry);
1000
    } else {
1001
        err = kvm_arch_insert_hw_breakpoint(addr, len, type);
1002
        if (err)
1003
            return err;
1004
    }
1005

    
1006
    for (env = first_cpu; env != NULL; env = env->next_cpu) {
1007
        err = kvm_update_guest_debug(env, 0);
1008
        if (err)
1009
            return err;
1010
    }
1011
    return 0;
1012
}
1013

    
1014
int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
1015
                          target_ulong len, int type)
1016
{
1017
    struct kvm_sw_breakpoint *bp;
1018
    CPUState *env;
1019
    int err;
1020

    
1021
    if (type == GDB_BREAKPOINT_SW) {
1022
        bp = kvm_find_sw_breakpoint(current_env, addr);
1023
        if (!bp)
1024
            return -ENOENT;
1025

    
1026
        if (bp->use_count > 1) {
1027
            bp->use_count--;
1028
            return 0;
1029
        }
1030

    
1031
        err = kvm_arch_remove_sw_breakpoint(current_env, bp);
1032
        if (err)
1033
            return err;
1034

    
1035
        QTAILQ_REMOVE(&current_env->kvm_state->kvm_sw_breakpoints, bp, entry);
1036
        qemu_free(bp);
1037
    } else {
1038
        err = kvm_arch_remove_hw_breakpoint(addr, len, type);
1039
        if (err)
1040
            return err;
1041
    }
1042

    
1043
    for (env = first_cpu; env != NULL; env = env->next_cpu) {
1044
        err = kvm_update_guest_debug(env, 0);
1045
        if (err)
1046
            return err;
1047
    }
1048
    return 0;
1049
}
1050

    
1051
void kvm_remove_all_breakpoints(CPUState *current_env)
1052
{
1053
    struct kvm_sw_breakpoint *bp, *next;
1054
    KVMState *s = current_env->kvm_state;
1055
    CPUState *env;
1056

    
1057
    QTAILQ_FOREACH_SAFE(bp, &s->kvm_sw_breakpoints, entry, next) {
1058
        if (kvm_arch_remove_sw_breakpoint(current_env, bp) != 0) {
1059
            /* Try harder to find a CPU that currently sees the breakpoint. */
1060
            for (env = first_cpu; env != NULL; env = env->next_cpu) {
1061
                if (kvm_arch_remove_sw_breakpoint(env, bp) == 0)
1062
                    break;
1063
            }
1064
        }
1065
    }
1066
    kvm_arch_remove_all_hw_breakpoints();
1067

    
1068
    for (env = first_cpu; env != NULL; env = env->next_cpu)
1069
        kvm_update_guest_debug(env, 0);
1070
}
1071

    
1072
#else /* !KVM_CAP_SET_GUEST_DEBUG */
1073

    
1074
int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
1075
{
1076
    return -EINVAL;
1077
}
1078

    
1079
int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
1080
                          target_ulong len, int type)
1081
{
1082
    return -EINVAL;
1083
}
1084

    
1085
int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
1086
                          target_ulong len, int type)
1087
{
1088
    return -EINVAL;
1089
}
1090

    
1091
void kvm_remove_all_breakpoints(CPUState *current_env)
1092
{
1093
}
1094
#endif /* !KVM_CAP_SET_GUEST_DEBUG */