Statistics
| Branch: | Revision:

root / kvm-all.c @ 8f6f962b

History | View | Annotate | Download (39.6 kB)

1 05330448 aliguori
/*
2 05330448 aliguori
 * QEMU KVM support
3 05330448 aliguori
 *
4 05330448 aliguori
 * Copyright IBM, Corp. 2008
5 5832d1f2 aliguori
 *           Red Hat, Inc. 2008
6 05330448 aliguori
 *
7 05330448 aliguori
 * Authors:
8 05330448 aliguori
 *  Anthony Liguori   <aliguori@us.ibm.com>
9 5832d1f2 aliguori
 *  Glauber Costa     <gcosta@redhat.com>
10 05330448 aliguori
 *
11 05330448 aliguori
 * This work is licensed under the terms of the GNU GPL, version 2 or later.
12 05330448 aliguori
 * See the COPYING file in the top-level directory.
13 05330448 aliguori
 *
14 05330448 aliguori
 */
15 05330448 aliguori
16 05330448 aliguori
#include <sys/types.h>
17 05330448 aliguori
#include <sys/ioctl.h>
18 05330448 aliguori
#include <sys/mman.h>
19 984b5181 aliguori
#include <stdarg.h>
20 05330448 aliguori
21 05330448 aliguori
#include <linux/kvm.h>
22 05330448 aliguori
23 05330448 aliguori
#include "qemu-common.h"
24 85199474 Marcelo Tosatti
#include "qemu-barrier.h"
25 05330448 aliguori
#include "sysemu.h"
26 d33a1810 Jan Kiszka
#include "hw/hw.h"
27 e22a25c9 aliguori
#include "gdbstub.h"
28 05330448 aliguori
#include "kvm.h"
29 8369e01c Marcelo Tosatti
#include "bswap.h"
30 a01672d3 Avi Kivity
#include "memory.h"
31 05330448 aliguori
32 d2f2b8a7 Stefan Hajnoczi
/* This check must be after config-host.h is included */
33 d2f2b8a7 Stefan Hajnoczi
#ifdef CONFIG_EVENTFD
34 d2f2b8a7 Stefan Hajnoczi
#include <sys/eventfd.h>
35 d2f2b8a7 Stefan Hajnoczi
#endif
36 d2f2b8a7 Stefan Hajnoczi
37 f65ed4c1 aliguori
/* KVM uses PAGE_SIZE in it's definition of COALESCED_MMIO_MAX */
38 f65ed4c1 aliguori
#define PAGE_SIZE TARGET_PAGE_SIZE
39 f65ed4c1 aliguori
40 05330448 aliguori
//#define DEBUG_KVM
41 05330448 aliguori
42 05330448 aliguori
#ifdef DEBUG_KVM
43 8c0d577e Blue Swirl
#define DPRINTF(fmt, ...) \
44 05330448 aliguori
    do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
45 05330448 aliguori
#else
46 8c0d577e Blue Swirl
#define DPRINTF(fmt, ...) \
47 05330448 aliguori
    do { } while (0)
48 05330448 aliguori
#endif
49 05330448 aliguori
50 34fc643f aliguori
typedef struct KVMSlot
51 34fc643f aliguori
{
52 c227f099 Anthony Liguori
    target_phys_addr_t start_addr;
53 c227f099 Anthony Liguori
    ram_addr_t memory_size;
54 9f213ed9 Avi Kivity
    void *ram;
55 34fc643f aliguori
    int slot;
56 34fc643f aliguori
    int flags;
57 34fc643f aliguori
} KVMSlot;
58 05330448 aliguori
59 5832d1f2 aliguori
typedef struct kvm_dirty_log KVMDirtyLog;
60 5832d1f2 aliguori
61 05330448 aliguori
struct KVMState
62 05330448 aliguori
{
63 05330448 aliguori
    KVMSlot slots[32];
64 05330448 aliguori
    int fd;
65 05330448 aliguori
    int vmfd;
66 f65ed4c1 aliguori
    int coalesced_mmio;
67 62a2744c Sheng Yang
    struct kvm_coalesced_mmio_ring *coalesced_mmio_ring;
68 1cae88b9 Avi Kivity
    bool coalesced_flush_in_progress;
69 e69917e2 Jan Kiszka
    int broken_set_mem_region;
70 4495d6a7 Jan Kiszka
    int migration_log;
71 a0fb002c Jan Kiszka
    int vcpu_events;
72 b0b1d690 Jan Kiszka
    int robust_singlestep;
73 ff44f1a3 Jan Kiszka
    int debugregs;
74 e22a25c9 aliguori
#ifdef KVM_CAP_SET_GUEST_DEBUG
75 e22a25c9 aliguori
    struct kvm_sw_breakpoint_head kvm_sw_breakpoints;
76 e22a25c9 aliguori
#endif
77 6f725c13 Glauber Costa
    int pit_in_kernel;
78 f1665b21 Sheng Yang
    int xsave, xcrs;
79 d2f2b8a7 Stefan Hajnoczi
    int many_ioeventfds;
80 84b058d7 Jan Kiszka
    int irqchip_inject_ioctl;
81 84b058d7 Jan Kiszka
#ifdef KVM_CAP_IRQ_ROUTING
82 84b058d7 Jan Kiszka
    struct kvm_irq_routing *irq_routes;
83 84b058d7 Jan Kiszka
    int nr_allocated_irq_routes;
84 84b058d7 Jan Kiszka
    uint32_t *used_gsi_bitmap;
85 84b058d7 Jan Kiszka
    unsigned int max_gsi;
86 84b058d7 Jan Kiszka
#endif
87 05330448 aliguori
};
88 05330448 aliguori
89 6a7af8cb Jan Kiszka
KVMState *kvm_state;
90 3d4b2649 Jan Kiszka
bool kvm_kernel_irqchip;
91 05330448 aliguori
92 94a8d39a Jan Kiszka
static const KVMCapabilityInfo kvm_required_capabilites[] = {
93 94a8d39a Jan Kiszka
    KVM_CAP_INFO(USER_MEMORY),
94 94a8d39a Jan Kiszka
    KVM_CAP_INFO(DESTROY_MEMORY_REGION_WORKS),
95 94a8d39a Jan Kiszka
    KVM_CAP_LAST_INFO
96 94a8d39a Jan Kiszka
};
97 94a8d39a Jan Kiszka
98 05330448 aliguori
static KVMSlot *kvm_alloc_slot(KVMState *s)
99 05330448 aliguori
{
100 05330448 aliguori
    int i;
101 05330448 aliguori
102 05330448 aliguori
    for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
103 a426e122 Jan Kiszka
        if (s->slots[i].memory_size == 0) {
104 05330448 aliguori
            return &s->slots[i];
105 a426e122 Jan Kiszka
        }
106 05330448 aliguori
    }
107 05330448 aliguori
108 d3f8d37f aliguori
    fprintf(stderr, "%s: no free slot available\n", __func__);
109 d3f8d37f aliguori
    abort();
110 d3f8d37f aliguori
}
111 d3f8d37f aliguori
112 d3f8d37f aliguori
static KVMSlot *kvm_lookup_matching_slot(KVMState *s,
113 c227f099 Anthony Liguori
                                         target_phys_addr_t start_addr,
114 c227f099 Anthony Liguori
                                         target_phys_addr_t end_addr)
115 d3f8d37f aliguori
{
116 d3f8d37f aliguori
    int i;
117 d3f8d37f aliguori
118 d3f8d37f aliguori
    for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
119 d3f8d37f aliguori
        KVMSlot *mem = &s->slots[i];
120 d3f8d37f aliguori
121 d3f8d37f aliguori
        if (start_addr == mem->start_addr &&
122 d3f8d37f aliguori
            end_addr == mem->start_addr + mem->memory_size) {
123 d3f8d37f aliguori
            return mem;
124 d3f8d37f aliguori
        }
125 d3f8d37f aliguori
    }
126 d3f8d37f aliguori
127 05330448 aliguori
    return NULL;
128 05330448 aliguori
}
129 05330448 aliguori
130 6152e2ae aliguori
/*
131 6152e2ae aliguori
 * Find overlapping slot with lowest start address
132 6152e2ae aliguori
 */
133 6152e2ae aliguori
static KVMSlot *kvm_lookup_overlapping_slot(KVMState *s,
134 c227f099 Anthony Liguori
                                            target_phys_addr_t start_addr,
135 c227f099 Anthony Liguori
                                            target_phys_addr_t end_addr)
136 05330448 aliguori
{
137 6152e2ae aliguori
    KVMSlot *found = NULL;
138 05330448 aliguori
    int i;
139 05330448 aliguori
140 05330448 aliguori
    for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
141 05330448 aliguori
        KVMSlot *mem = &s->slots[i];
142 05330448 aliguori
143 6152e2ae aliguori
        if (mem->memory_size == 0 ||
144 6152e2ae aliguori
            (found && found->start_addr < mem->start_addr)) {
145 6152e2ae aliguori
            continue;
146 6152e2ae aliguori
        }
147 6152e2ae aliguori
148 6152e2ae aliguori
        if (end_addr > mem->start_addr &&
149 6152e2ae aliguori
            start_addr < mem->start_addr + mem->memory_size) {
150 6152e2ae aliguori
            found = mem;
151 6152e2ae aliguori
        }
152 05330448 aliguori
    }
153 05330448 aliguori
154 6152e2ae aliguori
    return found;
155 05330448 aliguori
}
156 05330448 aliguori
157 9f213ed9 Avi Kivity
int kvm_physical_memory_addr_from_host(KVMState *s, void *ram,
158 9f213ed9 Avi Kivity
                                       target_phys_addr_t *phys_addr)
159 983dfc3b Huang Ying
{
160 983dfc3b Huang Ying
    int i;
161 983dfc3b Huang Ying
162 983dfc3b Huang Ying
    for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
163 983dfc3b Huang Ying
        KVMSlot *mem = &s->slots[i];
164 983dfc3b Huang Ying
165 9f213ed9 Avi Kivity
        if (ram >= mem->ram && ram < mem->ram + mem->memory_size) {
166 9f213ed9 Avi Kivity
            *phys_addr = mem->start_addr + (ram - mem->ram);
167 983dfc3b Huang Ying
            return 1;
168 983dfc3b Huang Ying
        }
169 983dfc3b Huang Ying
    }
170 983dfc3b Huang Ying
171 983dfc3b Huang Ying
    return 0;
172 983dfc3b Huang Ying
}
173 983dfc3b Huang Ying
174 5832d1f2 aliguori
static int kvm_set_user_memory_region(KVMState *s, KVMSlot *slot)
175 5832d1f2 aliguori
{
176 5832d1f2 aliguori
    struct kvm_userspace_memory_region mem;
177 5832d1f2 aliguori
178 5832d1f2 aliguori
    mem.slot = slot->slot;
179 5832d1f2 aliguori
    mem.guest_phys_addr = slot->start_addr;
180 5832d1f2 aliguori
    mem.memory_size = slot->memory_size;
181 9f213ed9 Avi Kivity
    mem.userspace_addr = (unsigned long)slot->ram;
182 5832d1f2 aliguori
    mem.flags = slot->flags;
183 4495d6a7 Jan Kiszka
    if (s->migration_log) {
184 4495d6a7 Jan Kiszka
        mem.flags |= KVM_MEM_LOG_DIRTY_PAGES;
185 4495d6a7 Jan Kiszka
    }
186 5832d1f2 aliguori
    return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
187 5832d1f2 aliguori
}
188 5832d1f2 aliguori
189 8d2ba1fb Jan Kiszka
static void kvm_reset_vcpu(void *opaque)
190 8d2ba1fb Jan Kiszka
{
191 8d2ba1fb Jan Kiszka
    CPUState *env = opaque;
192 8d2ba1fb Jan Kiszka
193 caa5af0f Jan Kiszka
    kvm_arch_reset_vcpu(env);
194 8d2ba1fb Jan Kiszka
}
195 5832d1f2 aliguori
196 6f725c13 Glauber Costa
int kvm_pit_in_kernel(void)
197 6f725c13 Glauber Costa
{
198 6f725c13 Glauber Costa
    return kvm_state->pit_in_kernel;
199 6f725c13 Glauber Costa
}
200 6f725c13 Glauber Costa
201 05330448 aliguori
int kvm_init_vcpu(CPUState *env)
202 05330448 aliguori
{
203 05330448 aliguori
    KVMState *s = kvm_state;
204 05330448 aliguori
    long mmap_size;
205 05330448 aliguori
    int ret;
206 05330448 aliguori
207 8c0d577e Blue Swirl
    DPRINTF("kvm_init_vcpu\n");
208 05330448 aliguori
209 984b5181 aliguori
    ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, env->cpu_index);
210 05330448 aliguori
    if (ret < 0) {
211 8c0d577e Blue Swirl
        DPRINTF("kvm_create_vcpu failed\n");
212 05330448 aliguori
        goto err;
213 05330448 aliguori
    }
214 05330448 aliguori
215 05330448 aliguori
    env->kvm_fd = ret;
216 05330448 aliguori
    env->kvm_state = s;
217 d841b6c4 Jan Kiszka
    env->kvm_vcpu_dirty = 1;
218 05330448 aliguori
219 05330448 aliguori
    mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
220 05330448 aliguori
    if (mmap_size < 0) {
221 748a680b Jan Kiszka
        ret = mmap_size;
222 8c0d577e Blue Swirl
        DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n");
223 05330448 aliguori
        goto err;
224 05330448 aliguori
    }
225 05330448 aliguori
226 05330448 aliguori
    env->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
227 05330448 aliguori
                        env->kvm_fd, 0);
228 05330448 aliguori
    if (env->kvm_run == MAP_FAILED) {
229 05330448 aliguori
        ret = -errno;
230 8c0d577e Blue Swirl
        DPRINTF("mmap'ing vcpu state failed\n");
231 05330448 aliguori
        goto err;
232 05330448 aliguori
    }
233 05330448 aliguori
234 a426e122 Jan Kiszka
    if (s->coalesced_mmio && !s->coalesced_mmio_ring) {
235 a426e122 Jan Kiszka
        s->coalesced_mmio_ring =
236 a426e122 Jan Kiszka
            (void *)env->kvm_run + s->coalesced_mmio * PAGE_SIZE;
237 a426e122 Jan Kiszka
    }
238 62a2744c Sheng Yang
239 05330448 aliguori
    ret = kvm_arch_init_vcpu(env);
240 8d2ba1fb Jan Kiszka
    if (ret == 0) {
241 a08d4367 Jan Kiszka
        qemu_register_reset(kvm_reset_vcpu, env);
242 caa5af0f Jan Kiszka
        kvm_arch_reset_vcpu(env);
243 8d2ba1fb Jan Kiszka
    }
244 05330448 aliguori
err:
245 05330448 aliguori
    return ret;
246 05330448 aliguori
}
247 05330448 aliguori
248 5832d1f2 aliguori
/*
249 5832d1f2 aliguori
 * dirty pages logging control
250 5832d1f2 aliguori
 */
251 25254bbc Michael S. Tsirkin
252 25254bbc Michael S. Tsirkin
static int kvm_mem_flags(KVMState *s, bool log_dirty)
253 25254bbc Michael S. Tsirkin
{
254 25254bbc Michael S. Tsirkin
    return log_dirty ? KVM_MEM_LOG_DIRTY_PAGES : 0;
255 25254bbc Michael S. Tsirkin
}
256 25254bbc Michael S. Tsirkin
257 25254bbc Michael S. Tsirkin
static int kvm_slot_dirty_pages_log_change(KVMSlot *mem, bool log_dirty)
258 5832d1f2 aliguori
{
259 5832d1f2 aliguori
    KVMState *s = kvm_state;
260 25254bbc Michael S. Tsirkin
    int flags, mask = KVM_MEM_LOG_DIRTY_PAGES;
261 4495d6a7 Jan Kiszka
    int old_flags;
262 4495d6a7 Jan Kiszka
263 4495d6a7 Jan Kiszka
    old_flags = mem->flags;
264 5832d1f2 aliguori
265 25254bbc Michael S. Tsirkin
    flags = (mem->flags & ~mask) | kvm_mem_flags(s, log_dirty);
266 5832d1f2 aliguori
    mem->flags = flags;
267 5832d1f2 aliguori
268 4495d6a7 Jan Kiszka
    /* If nothing changed effectively, no need to issue ioctl */
269 4495d6a7 Jan Kiszka
    if (s->migration_log) {
270 4495d6a7 Jan Kiszka
        flags |= KVM_MEM_LOG_DIRTY_PAGES;
271 4495d6a7 Jan Kiszka
    }
272 25254bbc Michael S. Tsirkin
273 4495d6a7 Jan Kiszka
    if (flags == old_flags) {
274 25254bbc Michael S. Tsirkin
        return 0;
275 4495d6a7 Jan Kiszka
    }
276 4495d6a7 Jan Kiszka
277 5832d1f2 aliguori
    return kvm_set_user_memory_region(s, mem);
278 5832d1f2 aliguori
}
279 5832d1f2 aliguori
280 25254bbc Michael S. Tsirkin
static int kvm_dirty_pages_log_change(target_phys_addr_t phys_addr,
281 25254bbc Michael S. Tsirkin
                                      ram_addr_t size, bool log_dirty)
282 25254bbc Michael S. Tsirkin
{
283 25254bbc Michael S. Tsirkin
    KVMState *s = kvm_state;
284 25254bbc Michael S. Tsirkin
    KVMSlot *mem = kvm_lookup_matching_slot(s, phys_addr, phys_addr + size);
285 25254bbc Michael S. Tsirkin
286 25254bbc Michael S. Tsirkin
    if (mem == NULL)  {
287 25254bbc Michael S. Tsirkin
        fprintf(stderr, "BUG: %s: invalid parameters " TARGET_FMT_plx "-"
288 25254bbc Michael S. Tsirkin
                TARGET_FMT_plx "\n", __func__, phys_addr,
289 25254bbc Michael S. Tsirkin
                (target_phys_addr_t)(phys_addr + size - 1));
290 25254bbc Michael S. Tsirkin
        return -EINVAL;
291 25254bbc Michael S. Tsirkin
    }
292 25254bbc Michael S. Tsirkin
    return kvm_slot_dirty_pages_log_change(mem, log_dirty);
293 25254bbc Michael S. Tsirkin
}
294 25254bbc Michael S. Tsirkin
295 a01672d3 Avi Kivity
static void kvm_log_start(MemoryListener *listener,
296 a01672d3 Avi Kivity
                          MemoryRegionSection *section)
297 5832d1f2 aliguori
{
298 a01672d3 Avi Kivity
    int r;
299 a01672d3 Avi Kivity
300 a01672d3 Avi Kivity
    r = kvm_dirty_pages_log_change(section->offset_within_address_space,
301 a01672d3 Avi Kivity
                                   section->size, true);
302 a01672d3 Avi Kivity
    if (r < 0) {
303 a01672d3 Avi Kivity
        abort();
304 a01672d3 Avi Kivity
    }
305 5832d1f2 aliguori
}
306 5832d1f2 aliguori
307 a01672d3 Avi Kivity
static void kvm_log_stop(MemoryListener *listener,
308 a01672d3 Avi Kivity
                          MemoryRegionSection *section)
309 5832d1f2 aliguori
{
310 a01672d3 Avi Kivity
    int r;
311 a01672d3 Avi Kivity
312 a01672d3 Avi Kivity
    r = kvm_dirty_pages_log_change(section->offset_within_address_space,
313 a01672d3 Avi Kivity
                                   section->size, false);
314 a01672d3 Avi Kivity
    if (r < 0) {
315 a01672d3 Avi Kivity
        abort();
316 a01672d3 Avi Kivity
    }
317 5832d1f2 aliguori
}
318 5832d1f2 aliguori
319 7b8f3b78 Michael S. Tsirkin
static int kvm_set_migration_log(int enable)
320 4495d6a7 Jan Kiszka
{
321 4495d6a7 Jan Kiszka
    KVMState *s = kvm_state;
322 4495d6a7 Jan Kiszka
    KVMSlot *mem;
323 4495d6a7 Jan Kiszka
    int i, err;
324 4495d6a7 Jan Kiszka
325 4495d6a7 Jan Kiszka
    s->migration_log = enable;
326 4495d6a7 Jan Kiszka
327 4495d6a7 Jan Kiszka
    for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
328 4495d6a7 Jan Kiszka
        mem = &s->slots[i];
329 4495d6a7 Jan Kiszka
330 70fedd76 Alex Williamson
        if (!mem->memory_size) {
331 70fedd76 Alex Williamson
            continue;
332 70fedd76 Alex Williamson
        }
333 4495d6a7 Jan Kiszka
        if (!!(mem->flags & KVM_MEM_LOG_DIRTY_PAGES) == enable) {
334 4495d6a7 Jan Kiszka
            continue;
335 4495d6a7 Jan Kiszka
        }
336 4495d6a7 Jan Kiszka
        err = kvm_set_user_memory_region(s, mem);
337 4495d6a7 Jan Kiszka
        if (err) {
338 4495d6a7 Jan Kiszka
            return err;
339 4495d6a7 Jan Kiszka
        }
340 4495d6a7 Jan Kiszka
    }
341 4495d6a7 Jan Kiszka
    return 0;
342 4495d6a7 Jan Kiszka
}
343 4495d6a7 Jan Kiszka
344 8369e01c Marcelo Tosatti
/* get kvm's dirty pages bitmap and update qemu's */
345 ffcde12f Avi Kivity
static int kvm_get_dirty_pages_log_range(MemoryRegionSection *section,
346 ffcde12f Avi Kivity
                                         unsigned long *bitmap)
347 96c1606b Alexander Graf
{
348 8369e01c Marcelo Tosatti
    unsigned int i, j;
349 aa90fec7 Benjamin Herrenschmidt
    unsigned long page_number, c;
350 aa90fec7 Benjamin Herrenschmidt
    target_phys_addr_t addr, addr1;
351 ffcde12f Avi Kivity
    unsigned int len = ((section->size / TARGET_PAGE_SIZE) + HOST_LONG_BITS - 1) / HOST_LONG_BITS;
352 8369e01c Marcelo Tosatti
353 8369e01c Marcelo Tosatti
    /*
354 8369e01c Marcelo Tosatti
     * bitmap-traveling is faster than memory-traveling (for addr...)
355 8369e01c Marcelo Tosatti
     * especially when most of the memory is not dirty.
356 8369e01c Marcelo Tosatti
     */
357 8369e01c Marcelo Tosatti
    for (i = 0; i < len; i++) {
358 8369e01c Marcelo Tosatti
        if (bitmap[i] != 0) {
359 8369e01c Marcelo Tosatti
            c = leul_to_cpu(bitmap[i]);
360 8369e01c Marcelo Tosatti
            do {
361 8369e01c Marcelo Tosatti
                j = ffsl(c) - 1;
362 8369e01c Marcelo Tosatti
                c &= ~(1ul << j);
363 8369e01c Marcelo Tosatti
                page_number = i * HOST_LONG_BITS + j;
364 8369e01c Marcelo Tosatti
                addr1 = page_number * TARGET_PAGE_SIZE;
365 ffcde12f Avi Kivity
                addr = section->offset_within_region + addr1;
366 fd4aa979 Blue Swirl
                memory_region_set_dirty(section->mr, addr, TARGET_PAGE_SIZE);
367 8369e01c Marcelo Tosatti
            } while (c != 0);
368 8369e01c Marcelo Tosatti
        }
369 8369e01c Marcelo Tosatti
    }
370 8369e01c Marcelo Tosatti
    return 0;
371 96c1606b Alexander Graf
}
372 96c1606b Alexander Graf
373 8369e01c Marcelo Tosatti
#define ALIGN(x, y)  (((x)+(y)-1) & ~((y)-1))
374 8369e01c Marcelo Tosatti
375 5832d1f2 aliguori
/**
376 5832d1f2 aliguori
 * kvm_physical_sync_dirty_bitmap - Grab dirty bitmap from kernel space
377 fd4aa979 Blue Swirl
 * This function updates qemu's dirty bitmap using
378 fd4aa979 Blue Swirl
 * memory_region_set_dirty().  This means all bits are set
379 fd4aa979 Blue Swirl
 * to dirty.
380 5832d1f2 aliguori
 *
381 d3f8d37f aliguori
 * @start_add: start of logged region.
382 5832d1f2 aliguori
 * @end_addr: end of logged region.
383 5832d1f2 aliguori
 */
384 ffcde12f Avi Kivity
static int kvm_physical_sync_dirty_bitmap(MemoryRegionSection *section)
385 5832d1f2 aliguori
{
386 5832d1f2 aliguori
    KVMState *s = kvm_state;
387 151f7749 Jan Kiszka
    unsigned long size, allocated_size = 0;
388 151f7749 Jan Kiszka
    KVMDirtyLog d;
389 151f7749 Jan Kiszka
    KVMSlot *mem;
390 151f7749 Jan Kiszka
    int ret = 0;
391 ffcde12f Avi Kivity
    target_phys_addr_t start_addr = section->offset_within_address_space;
392 ffcde12f Avi Kivity
    target_phys_addr_t end_addr = start_addr + section->size;
393 5832d1f2 aliguori
394 151f7749 Jan Kiszka
    d.dirty_bitmap = NULL;
395 151f7749 Jan Kiszka
    while (start_addr < end_addr) {
396 151f7749 Jan Kiszka
        mem = kvm_lookup_overlapping_slot(s, start_addr, end_addr);
397 151f7749 Jan Kiszka
        if (mem == NULL) {
398 151f7749 Jan Kiszka
            break;
399 151f7749 Jan Kiszka
        }
400 5832d1f2 aliguori
401 51b0c606 Michael Tokarev
        /* XXX bad kernel interface alert
402 51b0c606 Michael Tokarev
         * For dirty bitmap, kernel allocates array of size aligned to
403 51b0c606 Michael Tokarev
         * bits-per-long.  But for case when the kernel is 64bits and
404 51b0c606 Michael Tokarev
         * the userspace is 32bits, userspace can't align to the same
405 51b0c606 Michael Tokarev
         * bits-per-long, since sizeof(long) is different between kernel
406 51b0c606 Michael Tokarev
         * and user space.  This way, userspace will provide buffer which
407 51b0c606 Michael Tokarev
         * may be 4 bytes less than the kernel will use, resulting in
408 51b0c606 Michael Tokarev
         * userspace memory corruption (which is not detectable by valgrind
409 51b0c606 Michael Tokarev
         * too, in most cases).
410 51b0c606 Michael Tokarev
         * So for now, let's align to 64 instead of HOST_LONG_BITS here, in
411 51b0c606 Michael Tokarev
         * a hope that sizeof(long) wont become >8 any time soon.
412 51b0c606 Michael Tokarev
         */
413 51b0c606 Michael Tokarev
        size = ALIGN(((mem->memory_size) >> TARGET_PAGE_BITS),
414 51b0c606 Michael Tokarev
                     /*HOST_LONG_BITS*/ 64) / 8;
415 151f7749 Jan Kiszka
        if (!d.dirty_bitmap) {
416 7267c094 Anthony Liguori
            d.dirty_bitmap = g_malloc(size);
417 151f7749 Jan Kiszka
        } else if (size > allocated_size) {
418 7267c094 Anthony Liguori
            d.dirty_bitmap = g_realloc(d.dirty_bitmap, size);
419 151f7749 Jan Kiszka
        }
420 151f7749 Jan Kiszka
        allocated_size = size;
421 151f7749 Jan Kiszka
        memset(d.dirty_bitmap, 0, allocated_size);
422 5832d1f2 aliguori
423 151f7749 Jan Kiszka
        d.slot = mem->slot;
424 5832d1f2 aliguori
425 6e489f3f Anthony Liguori
        if (kvm_vm_ioctl(s, KVM_GET_DIRTY_LOG, &d) == -1) {
426 8c0d577e Blue Swirl
            DPRINTF("ioctl failed %d\n", errno);
427 151f7749 Jan Kiszka
            ret = -1;
428 151f7749 Jan Kiszka
            break;
429 151f7749 Jan Kiszka
        }
430 5832d1f2 aliguori
431 ffcde12f Avi Kivity
        kvm_get_dirty_pages_log_range(section, d.dirty_bitmap);
432 8369e01c Marcelo Tosatti
        start_addr = mem->start_addr + mem->memory_size;
433 5832d1f2 aliguori
    }
434 7267c094 Anthony Liguori
    g_free(d.dirty_bitmap);
435 151f7749 Jan Kiszka
436 151f7749 Jan Kiszka
    return ret;
437 5832d1f2 aliguori
}
438 5832d1f2 aliguori
439 c227f099 Anthony Liguori
int kvm_coalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
440 f65ed4c1 aliguori
{
441 f65ed4c1 aliguori
    int ret = -ENOSYS;
442 f65ed4c1 aliguori
    KVMState *s = kvm_state;
443 f65ed4c1 aliguori
444 f65ed4c1 aliguori
    if (s->coalesced_mmio) {
445 f65ed4c1 aliguori
        struct kvm_coalesced_mmio_zone zone;
446 f65ed4c1 aliguori
447 f65ed4c1 aliguori
        zone.addr = start;
448 f65ed4c1 aliguori
        zone.size = size;
449 f65ed4c1 aliguori
450 f65ed4c1 aliguori
        ret = kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
451 f65ed4c1 aliguori
    }
452 f65ed4c1 aliguori
453 f65ed4c1 aliguori
    return ret;
454 f65ed4c1 aliguori
}
455 f65ed4c1 aliguori
456 c227f099 Anthony Liguori
int kvm_uncoalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
457 f65ed4c1 aliguori
{
458 f65ed4c1 aliguori
    int ret = -ENOSYS;
459 f65ed4c1 aliguori
    KVMState *s = kvm_state;
460 f65ed4c1 aliguori
461 f65ed4c1 aliguori
    if (s->coalesced_mmio) {
462 f65ed4c1 aliguori
        struct kvm_coalesced_mmio_zone zone;
463 f65ed4c1 aliguori
464 f65ed4c1 aliguori
        zone.addr = start;
465 f65ed4c1 aliguori
        zone.size = size;
466 f65ed4c1 aliguori
467 f65ed4c1 aliguori
        ret = kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
468 f65ed4c1 aliguori
    }
469 f65ed4c1 aliguori
470 f65ed4c1 aliguori
    return ret;
471 f65ed4c1 aliguori
}
472 f65ed4c1 aliguori
473 ad7b8b33 Anthony Liguori
int kvm_check_extension(KVMState *s, unsigned int extension)
474 ad7b8b33 Anthony Liguori
{
475 ad7b8b33 Anthony Liguori
    int ret;
476 ad7b8b33 Anthony Liguori
477 ad7b8b33 Anthony Liguori
    ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, extension);
478 ad7b8b33 Anthony Liguori
    if (ret < 0) {
479 ad7b8b33 Anthony Liguori
        ret = 0;
480 ad7b8b33 Anthony Liguori
    }
481 ad7b8b33 Anthony Liguori
482 ad7b8b33 Anthony Liguori
    return ret;
483 ad7b8b33 Anthony Liguori
}
484 ad7b8b33 Anthony Liguori
485 d2f2b8a7 Stefan Hajnoczi
static int kvm_check_many_ioeventfds(void)
486 d2f2b8a7 Stefan Hajnoczi
{
487 d0dcac83 Stefan Hajnoczi
    /* Userspace can use ioeventfd for io notification.  This requires a host
488 d0dcac83 Stefan Hajnoczi
     * that supports eventfd(2) and an I/O thread; since eventfd does not
489 d0dcac83 Stefan Hajnoczi
     * support SIGIO it cannot interrupt the vcpu.
490 d0dcac83 Stefan Hajnoczi
     *
491 d0dcac83 Stefan Hajnoczi
     * Older kernels have a 6 device limit on the KVM io bus.  Find out so we
492 d2f2b8a7 Stefan Hajnoczi
     * can avoid creating too many ioeventfds.
493 d2f2b8a7 Stefan Hajnoczi
     */
494 12d4536f Anthony Liguori
#if defined(CONFIG_EVENTFD)
495 d2f2b8a7 Stefan Hajnoczi
    int ioeventfds[7];
496 d2f2b8a7 Stefan Hajnoczi
    int i, ret = 0;
497 d2f2b8a7 Stefan Hajnoczi
    for (i = 0; i < ARRAY_SIZE(ioeventfds); i++) {
498 d2f2b8a7 Stefan Hajnoczi
        ioeventfds[i] = eventfd(0, EFD_CLOEXEC);
499 d2f2b8a7 Stefan Hajnoczi
        if (ioeventfds[i] < 0) {
500 d2f2b8a7 Stefan Hajnoczi
            break;
501 d2f2b8a7 Stefan Hajnoczi
        }
502 d2f2b8a7 Stefan Hajnoczi
        ret = kvm_set_ioeventfd_pio_word(ioeventfds[i], 0, i, true);
503 d2f2b8a7 Stefan Hajnoczi
        if (ret < 0) {
504 d2f2b8a7 Stefan Hajnoczi
            close(ioeventfds[i]);
505 d2f2b8a7 Stefan Hajnoczi
            break;
506 d2f2b8a7 Stefan Hajnoczi
        }
507 d2f2b8a7 Stefan Hajnoczi
    }
508 d2f2b8a7 Stefan Hajnoczi
509 d2f2b8a7 Stefan Hajnoczi
    /* Decide whether many devices are supported or not */
510 d2f2b8a7 Stefan Hajnoczi
    ret = i == ARRAY_SIZE(ioeventfds);
511 d2f2b8a7 Stefan Hajnoczi
512 d2f2b8a7 Stefan Hajnoczi
    while (i-- > 0) {
513 d2f2b8a7 Stefan Hajnoczi
        kvm_set_ioeventfd_pio_word(ioeventfds[i], 0, i, false);
514 d2f2b8a7 Stefan Hajnoczi
        close(ioeventfds[i]);
515 d2f2b8a7 Stefan Hajnoczi
    }
516 d2f2b8a7 Stefan Hajnoczi
    return ret;
517 d2f2b8a7 Stefan Hajnoczi
#else
518 d2f2b8a7 Stefan Hajnoczi
    return 0;
519 d2f2b8a7 Stefan Hajnoczi
#endif
520 d2f2b8a7 Stefan Hajnoczi
}
521 d2f2b8a7 Stefan Hajnoczi
522 94a8d39a Jan Kiszka
static const KVMCapabilityInfo *
523 94a8d39a Jan Kiszka
kvm_check_extension_list(KVMState *s, const KVMCapabilityInfo *list)
524 94a8d39a Jan Kiszka
{
525 94a8d39a Jan Kiszka
    while (list->name) {
526 94a8d39a Jan Kiszka
        if (!kvm_check_extension(s, list->value)) {
527 94a8d39a Jan Kiszka
            return list;
528 94a8d39a Jan Kiszka
        }
529 94a8d39a Jan Kiszka
        list++;
530 94a8d39a Jan Kiszka
    }
531 94a8d39a Jan Kiszka
    return NULL;
532 94a8d39a Jan Kiszka
}
533 94a8d39a Jan Kiszka
534 a01672d3 Avi Kivity
static void kvm_set_phys_mem(MemoryRegionSection *section, bool add)
535 46dbef6a Michael S. Tsirkin
{
536 46dbef6a Michael S. Tsirkin
    KVMState *s = kvm_state;
537 46dbef6a Michael S. Tsirkin
    KVMSlot *mem, old;
538 46dbef6a Michael S. Tsirkin
    int err;
539 a01672d3 Avi Kivity
    MemoryRegion *mr = section->mr;
540 a01672d3 Avi Kivity
    bool log_dirty = memory_region_is_logging(mr);
541 a01672d3 Avi Kivity
    target_phys_addr_t start_addr = section->offset_within_address_space;
542 a01672d3 Avi Kivity
    ram_addr_t size = section->size;
543 9f213ed9 Avi Kivity
    void *ram = NULL;
544 8f6f962b Avi Kivity
    unsigned delta;
545 46dbef6a Michael S. Tsirkin
546 14542fea Gleb Natapov
    /* kvm works in page size chunks, but the function may be called
547 14542fea Gleb Natapov
       with sub-page size and unaligned start address. */
548 8f6f962b Avi Kivity
    delta = TARGET_PAGE_ALIGN(size) - size;
549 8f6f962b Avi Kivity
    if (delta > size) {
550 8f6f962b Avi Kivity
        return;
551 8f6f962b Avi Kivity
    }
552 8f6f962b Avi Kivity
    start_addr += delta;
553 8f6f962b Avi Kivity
    size -= delta;
554 8f6f962b Avi Kivity
    size &= TARGET_PAGE_MASK;
555 8f6f962b Avi Kivity
    if (!size || (start_addr & ~TARGET_PAGE_MASK)) {
556 8f6f962b Avi Kivity
        return;
557 8f6f962b Avi Kivity
    }
558 46dbef6a Michael S. Tsirkin
559 a01672d3 Avi Kivity
    if (!memory_region_is_ram(mr)) {
560 a01672d3 Avi Kivity
        return;
561 9f213ed9 Avi Kivity
    }
562 9f213ed9 Avi Kivity
563 8f6f962b Avi Kivity
    ram = memory_region_get_ram_ptr(mr) + section->offset_within_region + delta;
564 a01672d3 Avi Kivity
565 46dbef6a Michael S. Tsirkin
    while (1) {
566 46dbef6a Michael S. Tsirkin
        mem = kvm_lookup_overlapping_slot(s, start_addr, start_addr + size);
567 46dbef6a Michael S. Tsirkin
        if (!mem) {
568 46dbef6a Michael S. Tsirkin
            break;
569 46dbef6a Michael S. Tsirkin
        }
570 46dbef6a Michael S. Tsirkin
571 a01672d3 Avi Kivity
        if (add && start_addr >= mem->start_addr &&
572 46dbef6a Michael S. Tsirkin
            (start_addr + size <= mem->start_addr + mem->memory_size) &&
573 9f213ed9 Avi Kivity
            (ram - start_addr == mem->ram - mem->start_addr)) {
574 46dbef6a Michael S. Tsirkin
            /* The new slot fits into the existing one and comes with
575 25254bbc Michael S. Tsirkin
             * identical parameters - update flags and done. */
576 25254bbc Michael S. Tsirkin
            kvm_slot_dirty_pages_log_change(mem, log_dirty);
577 46dbef6a Michael S. Tsirkin
            return;
578 46dbef6a Michael S. Tsirkin
        }
579 46dbef6a Michael S. Tsirkin
580 46dbef6a Michael S. Tsirkin
        old = *mem;
581 46dbef6a Michael S. Tsirkin
582 3fbffb62 Avi Kivity
        if (mem->flags & KVM_MEM_LOG_DIRTY_PAGES) {
583 3fbffb62 Avi Kivity
            kvm_physical_sync_dirty_bitmap(section);
584 3fbffb62 Avi Kivity
        }
585 3fbffb62 Avi Kivity
586 46dbef6a Michael S. Tsirkin
        /* unregister the overlapping slot */
587 46dbef6a Michael S. Tsirkin
        mem->memory_size = 0;
588 46dbef6a Michael S. Tsirkin
        err = kvm_set_user_memory_region(s, mem);
589 46dbef6a Michael S. Tsirkin
        if (err) {
590 46dbef6a Michael S. Tsirkin
            fprintf(stderr, "%s: error unregistering overlapping slot: %s\n",
591 46dbef6a Michael S. Tsirkin
                    __func__, strerror(-err));
592 46dbef6a Michael S. Tsirkin
            abort();
593 46dbef6a Michael S. Tsirkin
        }
594 46dbef6a Michael S. Tsirkin
595 46dbef6a Michael S. Tsirkin
        /* Workaround for older KVM versions: we can't join slots, even not by
596 46dbef6a Michael S. Tsirkin
         * unregistering the previous ones and then registering the larger
597 46dbef6a Michael S. Tsirkin
         * slot. We have to maintain the existing fragmentation. Sigh.
598 46dbef6a Michael S. Tsirkin
         *
599 46dbef6a Michael S. Tsirkin
         * This workaround assumes that the new slot starts at the same
600 46dbef6a Michael S. Tsirkin
         * address as the first existing one. If not or if some overlapping
601 46dbef6a Michael S. Tsirkin
         * slot comes around later, we will fail (not seen in practice so far)
602 46dbef6a Michael S. Tsirkin
         * - and actually require a recent KVM version. */
603 46dbef6a Michael S. Tsirkin
        if (s->broken_set_mem_region &&
604 a01672d3 Avi Kivity
            old.start_addr == start_addr && old.memory_size < size && add) {
605 46dbef6a Michael S. Tsirkin
            mem = kvm_alloc_slot(s);
606 46dbef6a Michael S. Tsirkin
            mem->memory_size = old.memory_size;
607 46dbef6a Michael S. Tsirkin
            mem->start_addr = old.start_addr;
608 9f213ed9 Avi Kivity
            mem->ram = old.ram;
609 25254bbc Michael S. Tsirkin
            mem->flags = kvm_mem_flags(s, log_dirty);
610 46dbef6a Michael S. Tsirkin
611 46dbef6a Michael S. Tsirkin
            err = kvm_set_user_memory_region(s, mem);
612 46dbef6a Michael S. Tsirkin
            if (err) {
613 46dbef6a Michael S. Tsirkin
                fprintf(stderr, "%s: error updating slot: %s\n", __func__,
614 46dbef6a Michael S. Tsirkin
                        strerror(-err));
615 46dbef6a Michael S. Tsirkin
                abort();
616 46dbef6a Michael S. Tsirkin
            }
617 46dbef6a Michael S. Tsirkin
618 46dbef6a Michael S. Tsirkin
            start_addr += old.memory_size;
619 9f213ed9 Avi Kivity
            ram += old.memory_size;
620 46dbef6a Michael S. Tsirkin
            size -= old.memory_size;
621 46dbef6a Michael S. Tsirkin
            continue;
622 46dbef6a Michael S. Tsirkin
        }
623 46dbef6a Michael S. Tsirkin
624 46dbef6a Michael S. Tsirkin
        /* register prefix slot */
625 46dbef6a Michael S. Tsirkin
        if (old.start_addr < start_addr) {
626 46dbef6a Michael S. Tsirkin
            mem = kvm_alloc_slot(s);
627 46dbef6a Michael S. Tsirkin
            mem->memory_size = start_addr - old.start_addr;
628 46dbef6a Michael S. Tsirkin
            mem->start_addr = old.start_addr;
629 9f213ed9 Avi Kivity
            mem->ram = old.ram;
630 25254bbc Michael S. Tsirkin
            mem->flags =  kvm_mem_flags(s, log_dirty);
631 46dbef6a Michael S. Tsirkin
632 46dbef6a Michael S. Tsirkin
            err = kvm_set_user_memory_region(s, mem);
633 46dbef6a Michael S. Tsirkin
            if (err) {
634 46dbef6a Michael S. Tsirkin
                fprintf(stderr, "%s: error registering prefix slot: %s\n",
635 46dbef6a Michael S. Tsirkin
                        __func__, strerror(-err));
636 d4d6868f Alexander Graf
#ifdef TARGET_PPC
637 d4d6868f Alexander Graf
                fprintf(stderr, "%s: This is probably because your kernel's " \
638 d4d6868f Alexander Graf
                                "PAGE_SIZE is too big. Please try to use 4k " \
639 d4d6868f Alexander Graf
                                "PAGE_SIZE!\n", __func__);
640 d4d6868f Alexander Graf
#endif
641 46dbef6a Michael S. Tsirkin
                abort();
642 46dbef6a Michael S. Tsirkin
            }
643 46dbef6a Michael S. Tsirkin
        }
644 46dbef6a Michael S. Tsirkin
645 46dbef6a Michael S. Tsirkin
        /* register suffix slot */
646 46dbef6a Michael S. Tsirkin
        if (old.start_addr + old.memory_size > start_addr + size) {
647 46dbef6a Michael S. Tsirkin
            ram_addr_t size_delta;
648 46dbef6a Michael S. Tsirkin
649 46dbef6a Michael S. Tsirkin
            mem = kvm_alloc_slot(s);
650 46dbef6a Michael S. Tsirkin
            mem->start_addr = start_addr + size;
651 46dbef6a Michael S. Tsirkin
            size_delta = mem->start_addr - old.start_addr;
652 46dbef6a Michael S. Tsirkin
            mem->memory_size = old.memory_size - size_delta;
653 9f213ed9 Avi Kivity
            mem->ram = old.ram + size_delta;
654 25254bbc Michael S. Tsirkin
            mem->flags = kvm_mem_flags(s, log_dirty);
655 46dbef6a Michael S. Tsirkin
656 46dbef6a Michael S. Tsirkin
            err = kvm_set_user_memory_region(s, mem);
657 46dbef6a Michael S. Tsirkin
            if (err) {
658 46dbef6a Michael S. Tsirkin
                fprintf(stderr, "%s: error registering suffix slot: %s\n",
659 46dbef6a Michael S. Tsirkin
                        __func__, strerror(-err));
660 46dbef6a Michael S. Tsirkin
                abort();
661 46dbef6a Michael S. Tsirkin
            }
662 46dbef6a Michael S. Tsirkin
        }
663 46dbef6a Michael S. Tsirkin
    }
664 46dbef6a Michael S. Tsirkin
665 46dbef6a Michael S. Tsirkin
    /* in case the KVM bug workaround already "consumed" the new slot */
666 a426e122 Jan Kiszka
    if (!size) {
667 46dbef6a Michael S. Tsirkin
        return;
668 a426e122 Jan Kiszka
    }
669 a01672d3 Avi Kivity
    if (!add) {
670 46dbef6a Michael S. Tsirkin
        return;
671 a426e122 Jan Kiszka
    }
672 46dbef6a Michael S. Tsirkin
    mem = kvm_alloc_slot(s);
673 46dbef6a Michael S. Tsirkin
    mem->memory_size = size;
674 46dbef6a Michael S. Tsirkin
    mem->start_addr = start_addr;
675 9f213ed9 Avi Kivity
    mem->ram = ram;
676 25254bbc Michael S. Tsirkin
    mem->flags = kvm_mem_flags(s, log_dirty);
677 46dbef6a Michael S. Tsirkin
678 46dbef6a Michael S. Tsirkin
    err = kvm_set_user_memory_region(s, mem);
679 46dbef6a Michael S. Tsirkin
    if (err) {
680 46dbef6a Michael S. Tsirkin
        fprintf(stderr, "%s: error registering slot: %s\n", __func__,
681 46dbef6a Michael S. Tsirkin
                strerror(-err));
682 46dbef6a Michael S. Tsirkin
        abort();
683 46dbef6a Michael S. Tsirkin
    }
684 46dbef6a Michael S. Tsirkin
}
685 46dbef6a Michael S. Tsirkin
686 a01672d3 Avi Kivity
static void kvm_region_add(MemoryListener *listener,
687 a01672d3 Avi Kivity
                           MemoryRegionSection *section)
688 a01672d3 Avi Kivity
{
689 a01672d3 Avi Kivity
    kvm_set_phys_mem(section, true);
690 a01672d3 Avi Kivity
}
691 a01672d3 Avi Kivity
692 a01672d3 Avi Kivity
static void kvm_region_del(MemoryListener *listener,
693 a01672d3 Avi Kivity
                           MemoryRegionSection *section)
694 a01672d3 Avi Kivity
{
695 a01672d3 Avi Kivity
    kvm_set_phys_mem(section, false);
696 a01672d3 Avi Kivity
}
697 a01672d3 Avi Kivity
698 a01672d3 Avi Kivity
static void kvm_log_sync(MemoryListener *listener,
699 a01672d3 Avi Kivity
                         MemoryRegionSection *section)
700 7b8f3b78 Michael S. Tsirkin
{
701 a01672d3 Avi Kivity
    int r;
702 a01672d3 Avi Kivity
703 ffcde12f Avi Kivity
    r = kvm_physical_sync_dirty_bitmap(section);
704 a01672d3 Avi Kivity
    if (r < 0) {
705 a01672d3 Avi Kivity
        abort();
706 a01672d3 Avi Kivity
    }
707 7b8f3b78 Michael S. Tsirkin
}
708 7b8f3b78 Michael S. Tsirkin
709 a01672d3 Avi Kivity
static void kvm_log_global_start(struct MemoryListener *listener)
710 7b8f3b78 Michael S. Tsirkin
{
711 a01672d3 Avi Kivity
    int r;
712 a01672d3 Avi Kivity
713 a01672d3 Avi Kivity
    r = kvm_set_migration_log(1);
714 a01672d3 Avi Kivity
    assert(r >= 0);
715 7b8f3b78 Michael S. Tsirkin
}
716 7b8f3b78 Michael S. Tsirkin
717 a01672d3 Avi Kivity
static void kvm_log_global_stop(struct MemoryListener *listener)
718 7b8f3b78 Michael S. Tsirkin
{
719 a01672d3 Avi Kivity
    int r;
720 a01672d3 Avi Kivity
721 a01672d3 Avi Kivity
    r = kvm_set_migration_log(0);
722 a01672d3 Avi Kivity
    assert(r >= 0);
723 7b8f3b78 Michael S. Tsirkin
}
724 7b8f3b78 Michael S. Tsirkin
725 a01672d3 Avi Kivity
static MemoryListener kvm_memory_listener = {
726 a01672d3 Avi Kivity
    .region_add = kvm_region_add,
727 a01672d3 Avi Kivity
    .region_del = kvm_region_del,
728 e5896b12 Anthony PERARD
    .log_start = kvm_log_start,
729 e5896b12 Anthony PERARD
    .log_stop = kvm_log_stop,
730 a01672d3 Avi Kivity
    .log_sync = kvm_log_sync,
731 a01672d3 Avi Kivity
    .log_global_start = kvm_log_global_start,
732 a01672d3 Avi Kivity
    .log_global_stop = kvm_log_global_stop,
733 7b8f3b78 Michael S. Tsirkin
};
734 7b8f3b78 Michael S. Tsirkin
735 aa7f74d1 Jan Kiszka
static void kvm_handle_interrupt(CPUState *env, int mask)
736 aa7f74d1 Jan Kiszka
{
737 aa7f74d1 Jan Kiszka
    env->interrupt_request |= mask;
738 aa7f74d1 Jan Kiszka
739 aa7f74d1 Jan Kiszka
    if (!qemu_cpu_is_self(env)) {
740 aa7f74d1 Jan Kiszka
        qemu_cpu_kick(env);
741 aa7f74d1 Jan Kiszka
    }
742 aa7f74d1 Jan Kiszka
}
743 aa7f74d1 Jan Kiszka
744 84b058d7 Jan Kiszka
int kvm_irqchip_set_irq(KVMState *s, int irq, int level)
745 84b058d7 Jan Kiszka
{
746 84b058d7 Jan Kiszka
    struct kvm_irq_level event;
747 84b058d7 Jan Kiszka
    int ret;
748 84b058d7 Jan Kiszka
749 3d4b2649 Jan Kiszka
    assert(kvm_irqchip_in_kernel());
750 84b058d7 Jan Kiszka
751 84b058d7 Jan Kiszka
    event.level = level;
752 84b058d7 Jan Kiszka
    event.irq = irq;
753 84b058d7 Jan Kiszka
    ret = kvm_vm_ioctl(s, s->irqchip_inject_ioctl, &event);
754 84b058d7 Jan Kiszka
    if (ret < 0) {
755 84b058d7 Jan Kiszka
        perror("kvm_set_irqchip_line");
756 84b058d7 Jan Kiszka
        abort();
757 84b058d7 Jan Kiszka
    }
758 84b058d7 Jan Kiszka
759 84b058d7 Jan Kiszka
    return (s->irqchip_inject_ioctl == KVM_IRQ_LINE) ? 1 : event.status;
760 84b058d7 Jan Kiszka
}
761 84b058d7 Jan Kiszka
762 84b058d7 Jan Kiszka
#ifdef KVM_CAP_IRQ_ROUTING
763 84b058d7 Jan Kiszka
static void set_gsi(KVMState *s, unsigned int gsi)
764 84b058d7 Jan Kiszka
{
765 84b058d7 Jan Kiszka
    assert(gsi < s->max_gsi);
766 84b058d7 Jan Kiszka
767 84b058d7 Jan Kiszka
    s->used_gsi_bitmap[gsi / 32] |= 1U << (gsi % 32);
768 84b058d7 Jan Kiszka
}
769 84b058d7 Jan Kiszka
770 84b058d7 Jan Kiszka
static void kvm_init_irq_routing(KVMState *s)
771 84b058d7 Jan Kiszka
{
772 84b058d7 Jan Kiszka
    int gsi_count;
773 84b058d7 Jan Kiszka
774 84b058d7 Jan Kiszka
    gsi_count = kvm_check_extension(s, KVM_CAP_IRQ_ROUTING);
775 84b058d7 Jan Kiszka
    if (gsi_count > 0) {
776 84b058d7 Jan Kiszka
        unsigned int gsi_bits, i;
777 84b058d7 Jan Kiszka
778 84b058d7 Jan Kiszka
        /* Round up so we can search ints using ffs */
779 84b058d7 Jan Kiszka
        gsi_bits = (gsi_count + 31) / 32;
780 84b058d7 Jan Kiszka
        s->used_gsi_bitmap = g_malloc0(gsi_bits / 8);
781 84b058d7 Jan Kiszka
        s->max_gsi = gsi_bits;
782 84b058d7 Jan Kiszka
783 84b058d7 Jan Kiszka
        /* Mark any over-allocated bits as already in use */
784 84b058d7 Jan Kiszka
        for (i = gsi_count; i < gsi_bits; i++) {
785 84b058d7 Jan Kiszka
            set_gsi(s, i);
786 84b058d7 Jan Kiszka
        }
787 84b058d7 Jan Kiszka
    }
788 84b058d7 Jan Kiszka
789 84b058d7 Jan Kiszka
    s->irq_routes = g_malloc0(sizeof(*s->irq_routes));
790 84b058d7 Jan Kiszka
    s->nr_allocated_irq_routes = 0;
791 84b058d7 Jan Kiszka
792 84b058d7 Jan Kiszka
    kvm_arch_init_irq_routing(s);
793 84b058d7 Jan Kiszka
}
794 84b058d7 Jan Kiszka
795 84b058d7 Jan Kiszka
static void kvm_add_routing_entry(KVMState *s,
796 84b058d7 Jan Kiszka
                                  struct kvm_irq_routing_entry *entry)
797 84b058d7 Jan Kiszka
{
798 84b058d7 Jan Kiszka
    struct kvm_irq_routing_entry *new;
799 84b058d7 Jan Kiszka
    int n, size;
800 84b058d7 Jan Kiszka
801 84b058d7 Jan Kiszka
    if (s->irq_routes->nr == s->nr_allocated_irq_routes) {
802 84b058d7 Jan Kiszka
        n = s->nr_allocated_irq_routes * 2;
803 84b058d7 Jan Kiszka
        if (n < 64) {
804 84b058d7 Jan Kiszka
            n = 64;
805 84b058d7 Jan Kiszka
        }
806 84b058d7 Jan Kiszka
        size = sizeof(struct kvm_irq_routing);
807 84b058d7 Jan Kiszka
        size += n * sizeof(*new);
808 84b058d7 Jan Kiszka
        s->irq_routes = g_realloc(s->irq_routes, size);
809 84b058d7 Jan Kiszka
        s->nr_allocated_irq_routes = n;
810 84b058d7 Jan Kiszka
    }
811 84b058d7 Jan Kiszka
    n = s->irq_routes->nr++;
812 84b058d7 Jan Kiszka
    new = &s->irq_routes->entries[n];
813 84b058d7 Jan Kiszka
    memset(new, 0, sizeof(*new));
814 84b058d7 Jan Kiszka
    new->gsi = entry->gsi;
815 84b058d7 Jan Kiszka
    new->type = entry->type;
816 84b058d7 Jan Kiszka
    new->flags = entry->flags;
817 84b058d7 Jan Kiszka
    new->u = entry->u;
818 84b058d7 Jan Kiszka
819 84b058d7 Jan Kiszka
    set_gsi(s, entry->gsi);
820 84b058d7 Jan Kiszka
}
821 84b058d7 Jan Kiszka
822 84b058d7 Jan Kiszka
void kvm_irqchip_add_route(KVMState *s, int irq, int irqchip, int pin)
823 84b058d7 Jan Kiszka
{
824 84b058d7 Jan Kiszka
    struct kvm_irq_routing_entry e;
825 84b058d7 Jan Kiszka
826 84b058d7 Jan Kiszka
    e.gsi = irq;
827 84b058d7 Jan Kiszka
    e.type = KVM_IRQ_ROUTING_IRQCHIP;
828 84b058d7 Jan Kiszka
    e.flags = 0;
829 84b058d7 Jan Kiszka
    e.u.irqchip.irqchip = irqchip;
830 84b058d7 Jan Kiszka
    e.u.irqchip.pin = pin;
831 84b058d7 Jan Kiszka
    kvm_add_routing_entry(s, &e);
832 84b058d7 Jan Kiszka
}
833 84b058d7 Jan Kiszka
834 84b058d7 Jan Kiszka
int kvm_irqchip_commit_routes(KVMState *s)
835 84b058d7 Jan Kiszka
{
836 84b058d7 Jan Kiszka
    s->irq_routes->flags = 0;
837 84b058d7 Jan Kiszka
    return kvm_vm_ioctl(s, KVM_SET_GSI_ROUTING, s->irq_routes);
838 84b058d7 Jan Kiszka
}
839 84b058d7 Jan Kiszka
840 84b058d7 Jan Kiszka
#else /* !KVM_CAP_IRQ_ROUTING */
841 84b058d7 Jan Kiszka
842 84b058d7 Jan Kiszka
static void kvm_init_irq_routing(KVMState *s)
843 84b058d7 Jan Kiszka
{
844 84b058d7 Jan Kiszka
}
845 84b058d7 Jan Kiszka
#endif /* !KVM_CAP_IRQ_ROUTING */
846 84b058d7 Jan Kiszka
847 84b058d7 Jan Kiszka
static int kvm_irqchip_create(KVMState *s)
848 84b058d7 Jan Kiszka
{
849 84b058d7 Jan Kiszka
    QemuOptsList *list = qemu_find_opts("machine");
850 84b058d7 Jan Kiszka
    int ret;
851 84b058d7 Jan Kiszka
852 84b058d7 Jan Kiszka
    if (QTAILQ_EMPTY(&list->head) ||
853 84b058d7 Jan Kiszka
        !qemu_opt_get_bool(QTAILQ_FIRST(&list->head),
854 84b058d7 Jan Kiszka
                           "kernel_irqchip", false) ||
855 84b058d7 Jan Kiszka
        !kvm_check_extension(s, KVM_CAP_IRQCHIP)) {
856 84b058d7 Jan Kiszka
        return 0;
857 84b058d7 Jan Kiszka
    }
858 84b058d7 Jan Kiszka
859 84b058d7 Jan Kiszka
    ret = kvm_vm_ioctl(s, KVM_CREATE_IRQCHIP);
860 84b058d7 Jan Kiszka
    if (ret < 0) {
861 84b058d7 Jan Kiszka
        fprintf(stderr, "Create kernel irqchip failed\n");
862 84b058d7 Jan Kiszka
        return ret;
863 84b058d7 Jan Kiszka
    }
864 84b058d7 Jan Kiszka
865 84b058d7 Jan Kiszka
    s->irqchip_inject_ioctl = KVM_IRQ_LINE;
866 84b058d7 Jan Kiszka
    if (kvm_check_extension(s, KVM_CAP_IRQ_INJECT_STATUS)) {
867 84b058d7 Jan Kiszka
        s->irqchip_inject_ioctl = KVM_IRQ_LINE_STATUS;
868 84b058d7 Jan Kiszka
    }
869 3d4b2649 Jan Kiszka
    kvm_kernel_irqchip = true;
870 84b058d7 Jan Kiszka
871 84b058d7 Jan Kiszka
    kvm_init_irq_routing(s);
872 84b058d7 Jan Kiszka
873 84b058d7 Jan Kiszka
    return 0;
874 84b058d7 Jan Kiszka
}
875 84b058d7 Jan Kiszka
876 cad1e282 Jan Kiszka
int kvm_init(void)
877 05330448 aliguori
{
878 168ccc11 Jan Kiszka
    static const char upgrade_note[] =
879 168ccc11 Jan Kiszka
        "Please upgrade to at least kernel 2.6.29 or recent kvm-kmod\n"
880 168ccc11 Jan Kiszka
        "(see http://sourceforge.net/projects/kvm).\n";
881 05330448 aliguori
    KVMState *s;
882 94a8d39a Jan Kiszka
    const KVMCapabilityInfo *missing_cap;
883 05330448 aliguori
    int ret;
884 05330448 aliguori
    int i;
885 05330448 aliguori
886 7267c094 Anthony Liguori
    s = g_malloc0(sizeof(KVMState));
887 05330448 aliguori
888 e22a25c9 aliguori
#ifdef KVM_CAP_SET_GUEST_DEBUG
889 72cf2d4f Blue Swirl
    QTAILQ_INIT(&s->kvm_sw_breakpoints);
890 e22a25c9 aliguori
#endif
891 a426e122 Jan Kiszka
    for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
892 05330448 aliguori
        s->slots[i].slot = i;
893 a426e122 Jan Kiszka
    }
894 05330448 aliguori
    s->vmfd = -1;
895 40ff6d7e Kevin Wolf
    s->fd = qemu_open("/dev/kvm", O_RDWR);
896 05330448 aliguori
    if (s->fd == -1) {
897 05330448 aliguori
        fprintf(stderr, "Could not access KVM kernel module: %m\n");
898 05330448 aliguori
        ret = -errno;
899 05330448 aliguori
        goto err;
900 05330448 aliguori
    }
901 05330448 aliguori
902 05330448 aliguori
    ret = kvm_ioctl(s, KVM_GET_API_VERSION, 0);
903 05330448 aliguori
    if (ret < KVM_API_VERSION) {
904 a426e122 Jan Kiszka
        if (ret > 0) {
905 05330448 aliguori
            ret = -EINVAL;
906 a426e122 Jan Kiszka
        }
907 05330448 aliguori
        fprintf(stderr, "kvm version too old\n");
908 05330448 aliguori
        goto err;
909 05330448 aliguori
    }
910 05330448 aliguori
911 05330448 aliguori
    if (ret > KVM_API_VERSION) {
912 05330448 aliguori
        ret = -EINVAL;
913 05330448 aliguori
        fprintf(stderr, "kvm version not supported\n");
914 05330448 aliguori
        goto err;
915 05330448 aliguori
    }
916 05330448 aliguori
917 05330448 aliguori
    s->vmfd = kvm_ioctl(s, KVM_CREATE_VM, 0);
918 0104dcac Alexander Graf
    if (s->vmfd < 0) {
919 0104dcac Alexander Graf
#ifdef TARGET_S390X
920 0104dcac Alexander Graf
        fprintf(stderr, "Please add the 'switch_amode' kernel parameter to "
921 0104dcac Alexander Graf
                        "your host kernel command line\n");
922 0104dcac Alexander Graf
#endif
923 db9eae1c Xu He Jie
        ret = s->vmfd;
924 05330448 aliguori
        goto err;
925 0104dcac Alexander Graf
    }
926 05330448 aliguori
927 94a8d39a Jan Kiszka
    missing_cap = kvm_check_extension_list(s, kvm_required_capabilites);
928 94a8d39a Jan Kiszka
    if (!missing_cap) {
929 94a8d39a Jan Kiszka
        missing_cap =
930 94a8d39a Jan Kiszka
            kvm_check_extension_list(s, kvm_arch_required_capabilities);
931 05330448 aliguori
    }
932 94a8d39a Jan Kiszka
    if (missing_cap) {
933 ad7b8b33 Anthony Liguori
        ret = -EINVAL;
934 94a8d39a Jan Kiszka
        fprintf(stderr, "kvm does not support %s\n%s",
935 94a8d39a Jan Kiszka
                missing_cap->name, upgrade_note);
936 d85dc283 aliguori
        goto err;
937 d85dc283 aliguori
    }
938 d85dc283 aliguori
939 ad7b8b33 Anthony Liguori
    s->coalesced_mmio = kvm_check_extension(s, KVM_CAP_COALESCED_MMIO);
940 f65ed4c1 aliguori
941 e69917e2 Jan Kiszka
    s->broken_set_mem_region = 1;
942 14a09518 Lai Jiangshan
    ret = kvm_check_extension(s, KVM_CAP_JOIN_MEMORY_REGIONS_WORKS);
943 e69917e2 Jan Kiszka
    if (ret > 0) {
944 e69917e2 Jan Kiszka
        s->broken_set_mem_region = 0;
945 e69917e2 Jan Kiszka
    }
946 e69917e2 Jan Kiszka
947 a0fb002c Jan Kiszka
#ifdef KVM_CAP_VCPU_EVENTS
948 a0fb002c Jan Kiszka
    s->vcpu_events = kvm_check_extension(s, KVM_CAP_VCPU_EVENTS);
949 a0fb002c Jan Kiszka
#endif
950 a0fb002c Jan Kiszka
951 b0b1d690 Jan Kiszka
    s->robust_singlestep =
952 b0b1d690 Jan Kiszka
        kvm_check_extension(s, KVM_CAP_X86_ROBUST_SINGLESTEP);
953 b0b1d690 Jan Kiszka
954 ff44f1a3 Jan Kiszka
#ifdef KVM_CAP_DEBUGREGS
955 ff44f1a3 Jan Kiszka
    s->debugregs = kvm_check_extension(s, KVM_CAP_DEBUGREGS);
956 ff44f1a3 Jan Kiszka
#endif
957 ff44f1a3 Jan Kiszka
958 f1665b21 Sheng Yang
#ifdef KVM_CAP_XSAVE
959 f1665b21 Sheng Yang
    s->xsave = kvm_check_extension(s, KVM_CAP_XSAVE);
960 f1665b21 Sheng Yang
#endif
961 f1665b21 Sheng Yang
962 f1665b21 Sheng Yang
#ifdef KVM_CAP_XCRS
963 f1665b21 Sheng Yang
    s->xcrs = kvm_check_extension(s, KVM_CAP_XCRS);
964 f1665b21 Sheng Yang
#endif
965 f1665b21 Sheng Yang
966 cad1e282 Jan Kiszka
    ret = kvm_arch_init(s);
967 a426e122 Jan Kiszka
    if (ret < 0) {
968 05330448 aliguori
        goto err;
969 a426e122 Jan Kiszka
    }
970 05330448 aliguori
971 84b058d7 Jan Kiszka
    ret = kvm_irqchip_create(s);
972 84b058d7 Jan Kiszka
    if (ret < 0) {
973 84b058d7 Jan Kiszka
        goto err;
974 84b058d7 Jan Kiszka
    }
975 84b058d7 Jan Kiszka
976 05330448 aliguori
    kvm_state = s;
977 a01672d3 Avi Kivity
    memory_listener_register(&kvm_memory_listener);
978 05330448 aliguori
979 d2f2b8a7 Stefan Hajnoczi
    s->many_ioeventfds = kvm_check_many_ioeventfds();
980 d2f2b8a7 Stefan Hajnoczi
981 aa7f74d1 Jan Kiszka
    cpu_interrupt_handler = kvm_handle_interrupt;
982 aa7f74d1 Jan Kiszka
983 05330448 aliguori
    return 0;
984 05330448 aliguori
985 05330448 aliguori
err:
986 05330448 aliguori
    if (s) {
987 db9eae1c Xu He Jie
        if (s->vmfd >= 0) {
988 05330448 aliguori
            close(s->vmfd);
989 a426e122 Jan Kiszka
        }
990 a426e122 Jan Kiszka
        if (s->fd != -1) {
991 05330448 aliguori
            close(s->fd);
992 a426e122 Jan Kiszka
        }
993 05330448 aliguori
    }
994 7267c094 Anthony Liguori
    g_free(s);
995 05330448 aliguori
996 05330448 aliguori
    return ret;
997 05330448 aliguori
}
998 05330448 aliguori
999 b30e93e9 Jan Kiszka
static void kvm_handle_io(uint16_t port, void *data, int direction, int size,
1000 b30e93e9 Jan Kiszka
                          uint32_t count)
1001 05330448 aliguori
{
1002 05330448 aliguori
    int i;
1003 05330448 aliguori
    uint8_t *ptr = data;
1004 05330448 aliguori
1005 05330448 aliguori
    for (i = 0; i < count; i++) {
1006 05330448 aliguori
        if (direction == KVM_EXIT_IO_IN) {
1007 05330448 aliguori
            switch (size) {
1008 05330448 aliguori
            case 1:
1009 afcea8cb Blue Swirl
                stb_p(ptr, cpu_inb(port));
1010 05330448 aliguori
                break;
1011 05330448 aliguori
            case 2:
1012 afcea8cb Blue Swirl
                stw_p(ptr, cpu_inw(port));
1013 05330448 aliguori
                break;
1014 05330448 aliguori
            case 4:
1015 afcea8cb Blue Swirl
                stl_p(ptr, cpu_inl(port));
1016 05330448 aliguori
                break;
1017 05330448 aliguori
            }
1018 05330448 aliguori
        } else {
1019 05330448 aliguori
            switch (size) {
1020 05330448 aliguori
            case 1:
1021 afcea8cb Blue Swirl
                cpu_outb(port, ldub_p(ptr));
1022 05330448 aliguori
                break;
1023 05330448 aliguori
            case 2:
1024 afcea8cb Blue Swirl
                cpu_outw(port, lduw_p(ptr));
1025 05330448 aliguori
                break;
1026 05330448 aliguori
            case 4:
1027 afcea8cb Blue Swirl
                cpu_outl(port, ldl_p(ptr));
1028 05330448 aliguori
                break;
1029 05330448 aliguori
            }
1030 05330448 aliguori
        }
1031 05330448 aliguori
1032 05330448 aliguori
        ptr += size;
1033 05330448 aliguori
    }
1034 05330448 aliguori
}
1035 05330448 aliguori
1036 73aaec4a Jan Kiszka
static int kvm_handle_internal_error(CPUState *env, struct kvm_run *run)
1037 7c80eef8 Marcelo Tosatti
{
1038 bb44e0d1 Jan Kiszka
    fprintf(stderr, "KVM internal error.");
1039 7c80eef8 Marcelo Tosatti
    if (kvm_check_extension(kvm_state, KVM_CAP_INTERNAL_ERROR_DATA)) {
1040 7c80eef8 Marcelo Tosatti
        int i;
1041 7c80eef8 Marcelo Tosatti
1042 bb44e0d1 Jan Kiszka
        fprintf(stderr, " Suberror: %d\n", run->internal.suberror);
1043 7c80eef8 Marcelo Tosatti
        for (i = 0; i < run->internal.ndata; ++i) {
1044 7c80eef8 Marcelo Tosatti
            fprintf(stderr, "extra data[%d]: %"PRIx64"\n",
1045 7c80eef8 Marcelo Tosatti
                    i, (uint64_t)run->internal.data[i]);
1046 7c80eef8 Marcelo Tosatti
        }
1047 bb44e0d1 Jan Kiszka
    } else {
1048 bb44e0d1 Jan Kiszka
        fprintf(stderr, "\n");
1049 7c80eef8 Marcelo Tosatti
    }
1050 7c80eef8 Marcelo Tosatti
    if (run->internal.suberror == KVM_INTERNAL_ERROR_EMULATION) {
1051 7c80eef8 Marcelo Tosatti
        fprintf(stderr, "emulation failure\n");
1052 a426e122 Jan Kiszka
        if (!kvm_arch_stop_on_emulation_error(env)) {
1053 f5c848ee Jan Kiszka
            cpu_dump_state(env, stderr, fprintf, CPU_DUMP_CODE);
1054 d73cd8f4 Jan Kiszka
            return EXCP_INTERRUPT;
1055 a426e122 Jan Kiszka
        }
1056 7c80eef8 Marcelo Tosatti
    }
1057 7c80eef8 Marcelo Tosatti
    /* FIXME: Should trigger a qmp message to let management know
1058 7c80eef8 Marcelo Tosatti
     * something went wrong.
1059 7c80eef8 Marcelo Tosatti
     */
1060 73aaec4a Jan Kiszka
    return -1;
1061 7c80eef8 Marcelo Tosatti
}
1062 7c80eef8 Marcelo Tosatti
1063 62a2744c Sheng Yang
void kvm_flush_coalesced_mmio_buffer(void)
1064 f65ed4c1 aliguori
{
1065 f65ed4c1 aliguori
    KVMState *s = kvm_state;
1066 1cae88b9 Avi Kivity
1067 1cae88b9 Avi Kivity
    if (s->coalesced_flush_in_progress) {
1068 1cae88b9 Avi Kivity
        return;
1069 1cae88b9 Avi Kivity
    }
1070 1cae88b9 Avi Kivity
1071 1cae88b9 Avi Kivity
    s->coalesced_flush_in_progress = true;
1072 1cae88b9 Avi Kivity
1073 62a2744c Sheng Yang
    if (s->coalesced_mmio_ring) {
1074 62a2744c Sheng Yang
        struct kvm_coalesced_mmio_ring *ring = s->coalesced_mmio_ring;
1075 f65ed4c1 aliguori
        while (ring->first != ring->last) {
1076 f65ed4c1 aliguori
            struct kvm_coalesced_mmio *ent;
1077 f65ed4c1 aliguori
1078 f65ed4c1 aliguori
            ent = &ring->coalesced_mmio[ring->first];
1079 f65ed4c1 aliguori
1080 f65ed4c1 aliguori
            cpu_physical_memory_write(ent->phys_addr, ent->data, ent->len);
1081 85199474 Marcelo Tosatti
            smp_wmb();
1082 f65ed4c1 aliguori
            ring->first = (ring->first + 1) % KVM_COALESCED_MMIO_MAX;
1083 f65ed4c1 aliguori
        }
1084 f65ed4c1 aliguori
    }
1085 1cae88b9 Avi Kivity
1086 1cae88b9 Avi Kivity
    s->coalesced_flush_in_progress = false;
1087 f65ed4c1 aliguori
}
1088 f65ed4c1 aliguori
1089 2705d56a Jan Kiszka
static void do_kvm_cpu_synchronize_state(void *_env)
1090 4c0960c0 Avi Kivity
{
1091 2705d56a Jan Kiszka
    CPUState *env = _env;
1092 2705d56a Jan Kiszka
1093 9ded2744 Jan Kiszka
    if (!env->kvm_vcpu_dirty) {
1094 4c0960c0 Avi Kivity
        kvm_arch_get_registers(env);
1095 9ded2744 Jan Kiszka
        env->kvm_vcpu_dirty = 1;
1096 4c0960c0 Avi Kivity
    }
1097 4c0960c0 Avi Kivity
}
1098 4c0960c0 Avi Kivity
1099 2705d56a Jan Kiszka
void kvm_cpu_synchronize_state(CPUState *env)
1100 2705d56a Jan Kiszka
{
1101 a426e122 Jan Kiszka
    if (!env->kvm_vcpu_dirty) {
1102 2705d56a Jan Kiszka
        run_on_cpu(env, do_kvm_cpu_synchronize_state, env);
1103 a426e122 Jan Kiszka
    }
1104 2705d56a Jan Kiszka
}
1105 2705d56a Jan Kiszka
1106 ea375f9a Jan Kiszka
void kvm_cpu_synchronize_post_reset(CPUState *env)
1107 ea375f9a Jan Kiszka
{
1108 ea375f9a Jan Kiszka
    kvm_arch_put_registers(env, KVM_PUT_RESET_STATE);
1109 ea375f9a Jan Kiszka
    env->kvm_vcpu_dirty = 0;
1110 ea375f9a Jan Kiszka
}
1111 ea375f9a Jan Kiszka
1112 ea375f9a Jan Kiszka
void kvm_cpu_synchronize_post_init(CPUState *env)
1113 ea375f9a Jan Kiszka
{
1114 ea375f9a Jan Kiszka
    kvm_arch_put_registers(env, KVM_PUT_FULL_STATE);
1115 ea375f9a Jan Kiszka
    env->kvm_vcpu_dirty = 0;
1116 ea375f9a Jan Kiszka
}
1117 ea375f9a Jan Kiszka
1118 05330448 aliguori
int kvm_cpu_exec(CPUState *env)
1119 05330448 aliguori
{
1120 05330448 aliguori
    struct kvm_run *run = env->kvm_run;
1121 7cbb533f Jan Kiszka
    int ret, run_ret;
1122 05330448 aliguori
1123 8c0d577e Blue Swirl
    DPRINTF("kvm_cpu_exec()\n");
1124 05330448 aliguori
1125 99036865 Jan Kiszka
    if (kvm_arch_process_async_events(env)) {
1126 9ccfac9e Jan Kiszka
        env->exit_request = 0;
1127 6792a57b Jan Kiszka
        return EXCP_HLT;
1128 9ccfac9e Jan Kiszka
    }
1129 0af691d7 Marcelo Tosatti
1130 6792a57b Jan Kiszka
    cpu_single_env = env;
1131 6792a57b Jan Kiszka
1132 9ccfac9e Jan Kiszka
    do {
1133 9ded2744 Jan Kiszka
        if (env->kvm_vcpu_dirty) {
1134 ea375f9a Jan Kiszka
            kvm_arch_put_registers(env, KVM_PUT_RUNTIME_STATE);
1135 9ded2744 Jan Kiszka
            env->kvm_vcpu_dirty = 0;
1136 4c0960c0 Avi Kivity
        }
1137 4c0960c0 Avi Kivity
1138 8c14c173 Jan Kiszka
        kvm_arch_pre_run(env, run);
1139 9ccfac9e Jan Kiszka
        if (env->exit_request) {
1140 9ccfac9e Jan Kiszka
            DPRINTF("interrupt exit requested\n");
1141 9ccfac9e Jan Kiszka
            /*
1142 9ccfac9e Jan Kiszka
             * KVM requires us to reenter the kernel after IO exits to complete
1143 9ccfac9e Jan Kiszka
             * instruction emulation. This self-signal will ensure that we
1144 9ccfac9e Jan Kiszka
             * leave ASAP again.
1145 9ccfac9e Jan Kiszka
             */
1146 9ccfac9e Jan Kiszka
            qemu_cpu_kick_self();
1147 9ccfac9e Jan Kiszka
        }
1148 273faf1b Marcelo Tosatti
        cpu_single_env = NULL;
1149 d549db5a Glauber Costa
        qemu_mutex_unlock_iothread();
1150 9ccfac9e Jan Kiszka
1151 7cbb533f Jan Kiszka
        run_ret = kvm_vcpu_ioctl(env, KVM_RUN, 0);
1152 9ccfac9e Jan Kiszka
1153 d549db5a Glauber Costa
        qemu_mutex_lock_iothread();
1154 273faf1b Marcelo Tosatti
        cpu_single_env = env;
1155 05330448 aliguori
        kvm_arch_post_run(env, run);
1156 05330448 aliguori
1157 b0c883b5 Jan Kiszka
        kvm_flush_coalesced_mmio_buffer();
1158 b0c883b5 Jan Kiszka
1159 7cbb533f Jan Kiszka
        if (run_ret < 0) {
1160 dc77d341 Jan Kiszka
            if (run_ret == -EINTR || run_ret == -EAGAIN) {
1161 dc77d341 Jan Kiszka
                DPRINTF("io window exit\n");
1162 d73cd8f4 Jan Kiszka
                ret = EXCP_INTERRUPT;
1163 dc77d341 Jan Kiszka
                break;
1164 dc77d341 Jan Kiszka
            }
1165 7b011fbc Michael Ellerman
            fprintf(stderr, "error: kvm run failed %s\n",
1166 7b011fbc Michael Ellerman
                    strerror(-run_ret));
1167 05330448 aliguori
            abort();
1168 05330448 aliguori
        }
1169 05330448 aliguori
1170 05330448 aliguori
        switch (run->exit_reason) {
1171 05330448 aliguori
        case KVM_EXIT_IO:
1172 8c0d577e Blue Swirl
            DPRINTF("handle_io\n");
1173 b30e93e9 Jan Kiszka
            kvm_handle_io(run->io.port,
1174 b30e93e9 Jan Kiszka
                          (uint8_t *)run + run->io.data_offset,
1175 b30e93e9 Jan Kiszka
                          run->io.direction,
1176 b30e93e9 Jan Kiszka
                          run->io.size,
1177 b30e93e9 Jan Kiszka
                          run->io.count);
1178 d73cd8f4 Jan Kiszka
            ret = 0;
1179 05330448 aliguori
            break;
1180 05330448 aliguori
        case KVM_EXIT_MMIO:
1181 8c0d577e Blue Swirl
            DPRINTF("handle_mmio\n");
1182 05330448 aliguori
            cpu_physical_memory_rw(run->mmio.phys_addr,
1183 05330448 aliguori
                                   run->mmio.data,
1184 05330448 aliguori
                                   run->mmio.len,
1185 05330448 aliguori
                                   run->mmio.is_write);
1186 d73cd8f4 Jan Kiszka
            ret = 0;
1187 05330448 aliguori
            break;
1188 05330448 aliguori
        case KVM_EXIT_IRQ_WINDOW_OPEN:
1189 8c0d577e Blue Swirl
            DPRINTF("irq_window_open\n");
1190 d73cd8f4 Jan Kiszka
            ret = EXCP_INTERRUPT;
1191 05330448 aliguori
            break;
1192 05330448 aliguori
        case KVM_EXIT_SHUTDOWN:
1193 8c0d577e Blue Swirl
            DPRINTF("shutdown\n");
1194 05330448 aliguori
            qemu_system_reset_request();
1195 d73cd8f4 Jan Kiszka
            ret = EXCP_INTERRUPT;
1196 05330448 aliguori
            break;
1197 05330448 aliguori
        case KVM_EXIT_UNKNOWN:
1198 bb44e0d1 Jan Kiszka
            fprintf(stderr, "KVM: unknown exit, hardware reason %" PRIx64 "\n",
1199 bb44e0d1 Jan Kiszka
                    (uint64_t)run->hw.hardware_exit_reason);
1200 73aaec4a Jan Kiszka
            ret = -1;
1201 05330448 aliguori
            break;
1202 7c80eef8 Marcelo Tosatti
        case KVM_EXIT_INTERNAL_ERROR:
1203 73aaec4a Jan Kiszka
            ret = kvm_handle_internal_error(env, run);
1204 7c80eef8 Marcelo Tosatti
            break;
1205 05330448 aliguori
        default:
1206 8c0d577e Blue Swirl
            DPRINTF("kvm_arch_handle_exit\n");
1207 05330448 aliguori
            ret = kvm_arch_handle_exit(env, run);
1208 05330448 aliguori
            break;
1209 05330448 aliguori
        }
1210 d73cd8f4 Jan Kiszka
    } while (ret == 0);
1211 05330448 aliguori
1212 73aaec4a Jan Kiszka
    if (ret < 0) {
1213 f5c848ee Jan Kiszka
        cpu_dump_state(env, stderr, fprintf, CPU_DUMP_CODE);
1214 0461d5a6 Luiz Capitulino
        vm_stop(RUN_STATE_INTERNAL_ERROR);
1215 becfc390 aliguori
    }
1216 becfc390 aliguori
1217 6792a57b Jan Kiszka
    env->exit_request = 0;
1218 6792a57b Jan Kiszka
    cpu_single_env = NULL;
1219 05330448 aliguori
    return ret;
1220 05330448 aliguori
}
1221 05330448 aliguori
1222 984b5181 aliguori
int kvm_ioctl(KVMState *s, int type, ...)
1223 05330448 aliguori
{
1224 05330448 aliguori
    int ret;
1225 984b5181 aliguori
    void *arg;
1226 984b5181 aliguori
    va_list ap;
1227 05330448 aliguori
1228 984b5181 aliguori
    va_start(ap, type);
1229 984b5181 aliguori
    arg = va_arg(ap, void *);
1230 984b5181 aliguori
    va_end(ap);
1231 984b5181 aliguori
1232 984b5181 aliguori
    ret = ioctl(s->fd, type, arg);
1233 a426e122 Jan Kiszka
    if (ret == -1) {
1234 05330448 aliguori
        ret = -errno;
1235 a426e122 Jan Kiszka
    }
1236 05330448 aliguori
    return ret;
1237 05330448 aliguori
}
1238 05330448 aliguori
1239 984b5181 aliguori
int kvm_vm_ioctl(KVMState *s, int type, ...)
1240 05330448 aliguori
{
1241 05330448 aliguori
    int ret;
1242 984b5181 aliguori
    void *arg;
1243 984b5181 aliguori
    va_list ap;
1244 984b5181 aliguori
1245 984b5181 aliguori
    va_start(ap, type);
1246 984b5181 aliguori
    arg = va_arg(ap, void *);
1247 984b5181 aliguori
    va_end(ap);
1248 05330448 aliguori
1249 984b5181 aliguori
    ret = ioctl(s->vmfd, type, arg);
1250 a426e122 Jan Kiszka
    if (ret == -1) {
1251 05330448 aliguori
        ret = -errno;
1252 a426e122 Jan Kiszka
    }
1253 05330448 aliguori
    return ret;
1254 05330448 aliguori
}
1255 05330448 aliguori
1256 984b5181 aliguori
int kvm_vcpu_ioctl(CPUState *env, int type, ...)
1257 05330448 aliguori
{
1258 05330448 aliguori
    int ret;
1259 984b5181 aliguori
    void *arg;
1260 984b5181 aliguori
    va_list ap;
1261 984b5181 aliguori
1262 984b5181 aliguori
    va_start(ap, type);
1263 984b5181 aliguori
    arg = va_arg(ap, void *);
1264 984b5181 aliguori
    va_end(ap);
1265 05330448 aliguori
1266 984b5181 aliguori
    ret = ioctl(env->kvm_fd, type, arg);
1267 a426e122 Jan Kiszka
    if (ret == -1) {
1268 05330448 aliguori
        ret = -errno;
1269 a426e122 Jan Kiszka
    }
1270 05330448 aliguori
    return ret;
1271 05330448 aliguori
}
1272 bd322087 aliguori
1273 bd322087 aliguori
int kvm_has_sync_mmu(void)
1274 bd322087 aliguori
{
1275 94a8d39a Jan Kiszka
    return kvm_check_extension(kvm_state, KVM_CAP_SYNC_MMU);
1276 bd322087 aliguori
}
1277 e22a25c9 aliguori
1278 a0fb002c Jan Kiszka
int kvm_has_vcpu_events(void)
1279 a0fb002c Jan Kiszka
{
1280 a0fb002c Jan Kiszka
    return kvm_state->vcpu_events;
1281 a0fb002c Jan Kiszka
}
1282 a0fb002c Jan Kiszka
1283 b0b1d690 Jan Kiszka
int kvm_has_robust_singlestep(void)
1284 b0b1d690 Jan Kiszka
{
1285 b0b1d690 Jan Kiszka
    return kvm_state->robust_singlestep;
1286 b0b1d690 Jan Kiszka
}
1287 b0b1d690 Jan Kiszka
1288 ff44f1a3 Jan Kiszka
int kvm_has_debugregs(void)
1289 ff44f1a3 Jan Kiszka
{
1290 ff44f1a3 Jan Kiszka
    return kvm_state->debugregs;
1291 ff44f1a3 Jan Kiszka
}
1292 ff44f1a3 Jan Kiszka
1293 f1665b21 Sheng Yang
int kvm_has_xsave(void)
1294 f1665b21 Sheng Yang
{
1295 f1665b21 Sheng Yang
    return kvm_state->xsave;
1296 f1665b21 Sheng Yang
}
1297 f1665b21 Sheng Yang
1298 f1665b21 Sheng Yang
int kvm_has_xcrs(void)
1299 f1665b21 Sheng Yang
{
1300 f1665b21 Sheng Yang
    return kvm_state->xcrs;
1301 f1665b21 Sheng Yang
}
1302 f1665b21 Sheng Yang
1303 d2f2b8a7 Stefan Hajnoczi
int kvm_has_many_ioeventfds(void)
1304 d2f2b8a7 Stefan Hajnoczi
{
1305 d2f2b8a7 Stefan Hajnoczi
    if (!kvm_enabled()) {
1306 d2f2b8a7 Stefan Hajnoczi
        return 0;
1307 d2f2b8a7 Stefan Hajnoczi
    }
1308 d2f2b8a7 Stefan Hajnoczi
    return kvm_state->many_ioeventfds;
1309 d2f2b8a7 Stefan Hajnoczi
}
1310 d2f2b8a7 Stefan Hajnoczi
1311 84b058d7 Jan Kiszka
int kvm_has_gsi_routing(void)
1312 84b058d7 Jan Kiszka
{
1313 a9c5eb0d Alexander Graf
#ifdef KVM_CAP_IRQ_ROUTING
1314 84b058d7 Jan Kiszka
    return kvm_check_extension(kvm_state, KVM_CAP_IRQ_ROUTING);
1315 a9c5eb0d Alexander Graf
#else
1316 a9c5eb0d Alexander Graf
    return false;
1317 a9c5eb0d Alexander Graf
#endif
1318 84b058d7 Jan Kiszka
}
1319 84b058d7 Jan Kiszka
1320 9b5b76d4 Jan Kiszka
int kvm_allows_irq0_override(void)
1321 9b5b76d4 Jan Kiszka
{
1322 3d4b2649 Jan Kiszka
    return !kvm_irqchip_in_kernel() || kvm_has_gsi_routing();
1323 9b5b76d4 Jan Kiszka
}
1324 9b5b76d4 Jan Kiszka
1325 6f0437e8 Jan Kiszka
void kvm_setup_guest_memory(void *start, size_t size)
1326 6f0437e8 Jan Kiszka
{
1327 6f0437e8 Jan Kiszka
    if (!kvm_has_sync_mmu()) {
1328 e78815a5 Andreas Fรคrber
        int ret = qemu_madvise(start, size, QEMU_MADV_DONTFORK);
1329 6f0437e8 Jan Kiszka
1330 6f0437e8 Jan Kiszka
        if (ret) {
1331 e78815a5 Andreas Fรคrber
            perror("qemu_madvise");
1332 e78815a5 Andreas Fรคrber
            fprintf(stderr,
1333 e78815a5 Andreas Fรคrber
                    "Need MADV_DONTFORK in absence of synchronous KVM MMU\n");
1334 6f0437e8 Jan Kiszka
            exit(1);
1335 6f0437e8 Jan Kiszka
        }
1336 6f0437e8 Jan Kiszka
    }
1337 6f0437e8 Jan Kiszka
}
1338 6f0437e8 Jan Kiszka
1339 e22a25c9 aliguori
#ifdef KVM_CAP_SET_GUEST_DEBUG
1340 e22a25c9 aliguori
struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *env,
1341 e22a25c9 aliguori
                                                 target_ulong pc)
1342 e22a25c9 aliguori
{
1343 e22a25c9 aliguori
    struct kvm_sw_breakpoint *bp;
1344 e22a25c9 aliguori
1345 72cf2d4f Blue Swirl
    QTAILQ_FOREACH(bp, &env->kvm_state->kvm_sw_breakpoints, entry) {
1346 a426e122 Jan Kiszka
        if (bp->pc == pc) {
1347 e22a25c9 aliguori
            return bp;
1348 a426e122 Jan Kiszka
        }
1349 e22a25c9 aliguori
    }
1350 e22a25c9 aliguori
    return NULL;
1351 e22a25c9 aliguori
}
1352 e22a25c9 aliguori
1353 e22a25c9 aliguori
int kvm_sw_breakpoints_active(CPUState *env)
1354 e22a25c9 aliguori
{
1355 72cf2d4f Blue Swirl
    return !QTAILQ_EMPTY(&env->kvm_state->kvm_sw_breakpoints);
1356 e22a25c9 aliguori
}
1357 e22a25c9 aliguori
1358 452e4751 Glauber Costa
struct kvm_set_guest_debug_data {
1359 452e4751 Glauber Costa
    struct kvm_guest_debug dbg;
1360 452e4751 Glauber Costa
    CPUState *env;
1361 452e4751 Glauber Costa
    int err;
1362 452e4751 Glauber Costa
};
1363 452e4751 Glauber Costa
1364 452e4751 Glauber Costa
static void kvm_invoke_set_guest_debug(void *data)
1365 452e4751 Glauber Costa
{
1366 452e4751 Glauber Costa
    struct kvm_set_guest_debug_data *dbg_data = data;
1367 b3807725 Jan Kiszka
    CPUState *env = dbg_data->env;
1368 b3807725 Jan Kiszka
1369 b3807725 Jan Kiszka
    dbg_data->err = kvm_vcpu_ioctl(env, KVM_SET_GUEST_DEBUG, &dbg_data->dbg);
1370 452e4751 Glauber Costa
}
1371 452e4751 Glauber Costa
1372 e22a25c9 aliguori
int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
1373 e22a25c9 aliguori
{
1374 452e4751 Glauber Costa
    struct kvm_set_guest_debug_data data;
1375 e22a25c9 aliguori
1376 b0b1d690 Jan Kiszka
    data.dbg.control = reinject_trap;
1377 e22a25c9 aliguori
1378 b0b1d690 Jan Kiszka
    if (env->singlestep_enabled) {
1379 b0b1d690 Jan Kiszka
        data.dbg.control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP;
1380 b0b1d690 Jan Kiszka
    }
1381 452e4751 Glauber Costa
    kvm_arch_update_guest_debug(env, &data.dbg);
1382 452e4751 Glauber Costa
    data.env = env;
1383 e22a25c9 aliguori
1384 be41cbe0 Jan Kiszka
    run_on_cpu(env, kvm_invoke_set_guest_debug, &data);
1385 452e4751 Glauber Costa
    return data.err;
1386 e22a25c9 aliguori
}
1387 e22a25c9 aliguori
1388 e22a25c9 aliguori
int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
1389 e22a25c9 aliguori
                          target_ulong len, int type)
1390 e22a25c9 aliguori
{
1391 e22a25c9 aliguori
    struct kvm_sw_breakpoint *bp;
1392 e22a25c9 aliguori
    CPUState *env;
1393 e22a25c9 aliguori
    int err;
1394 e22a25c9 aliguori
1395 e22a25c9 aliguori
    if (type == GDB_BREAKPOINT_SW) {
1396 e22a25c9 aliguori
        bp = kvm_find_sw_breakpoint(current_env, addr);
1397 e22a25c9 aliguori
        if (bp) {
1398 e22a25c9 aliguori
            bp->use_count++;
1399 e22a25c9 aliguori
            return 0;
1400 e22a25c9 aliguori
        }
1401 e22a25c9 aliguori
1402 7267c094 Anthony Liguori
        bp = g_malloc(sizeof(struct kvm_sw_breakpoint));
1403 a426e122 Jan Kiszka
        if (!bp) {
1404 e22a25c9 aliguori
            return -ENOMEM;
1405 a426e122 Jan Kiszka
        }
1406 e22a25c9 aliguori
1407 e22a25c9 aliguori
        bp->pc = addr;
1408 e22a25c9 aliguori
        bp->use_count = 1;
1409 e22a25c9 aliguori
        err = kvm_arch_insert_sw_breakpoint(current_env, bp);
1410 e22a25c9 aliguori
        if (err) {
1411 7267c094 Anthony Liguori
            g_free(bp);
1412 e22a25c9 aliguori
            return err;
1413 e22a25c9 aliguori
        }
1414 e22a25c9 aliguori
1415 72cf2d4f Blue Swirl
        QTAILQ_INSERT_HEAD(&current_env->kvm_state->kvm_sw_breakpoints,
1416 e22a25c9 aliguori
                          bp, entry);
1417 e22a25c9 aliguori
    } else {
1418 e22a25c9 aliguori
        err = kvm_arch_insert_hw_breakpoint(addr, len, type);
1419 a426e122 Jan Kiszka
        if (err) {
1420 e22a25c9 aliguori
            return err;
1421 a426e122 Jan Kiszka
        }
1422 e22a25c9 aliguori
    }
1423 e22a25c9 aliguori
1424 e22a25c9 aliguori
    for (env = first_cpu; env != NULL; env = env->next_cpu) {
1425 e22a25c9 aliguori
        err = kvm_update_guest_debug(env, 0);
1426 a426e122 Jan Kiszka
        if (err) {
1427 e22a25c9 aliguori
            return err;
1428 a426e122 Jan Kiszka
        }
1429 e22a25c9 aliguori
    }
1430 e22a25c9 aliguori
    return 0;
1431 e22a25c9 aliguori
}
1432 e22a25c9 aliguori
1433 e22a25c9 aliguori
int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
1434 e22a25c9 aliguori
                          target_ulong len, int type)
1435 e22a25c9 aliguori
{
1436 e22a25c9 aliguori
    struct kvm_sw_breakpoint *bp;
1437 e22a25c9 aliguori
    CPUState *env;
1438 e22a25c9 aliguori
    int err;
1439 e22a25c9 aliguori
1440 e22a25c9 aliguori
    if (type == GDB_BREAKPOINT_SW) {
1441 e22a25c9 aliguori
        bp = kvm_find_sw_breakpoint(current_env, addr);
1442 a426e122 Jan Kiszka
        if (!bp) {
1443 e22a25c9 aliguori
            return -ENOENT;
1444 a426e122 Jan Kiszka
        }
1445 e22a25c9 aliguori
1446 e22a25c9 aliguori
        if (bp->use_count > 1) {
1447 e22a25c9 aliguori
            bp->use_count--;
1448 e22a25c9 aliguori
            return 0;
1449 e22a25c9 aliguori
        }
1450 e22a25c9 aliguori
1451 e22a25c9 aliguori
        err = kvm_arch_remove_sw_breakpoint(current_env, bp);
1452 a426e122 Jan Kiszka
        if (err) {
1453 e22a25c9 aliguori
            return err;
1454 a426e122 Jan Kiszka
        }
1455 e22a25c9 aliguori
1456 72cf2d4f Blue Swirl
        QTAILQ_REMOVE(&current_env->kvm_state->kvm_sw_breakpoints, bp, entry);
1457 7267c094 Anthony Liguori
        g_free(bp);
1458 e22a25c9 aliguori
    } else {
1459 e22a25c9 aliguori
        err = kvm_arch_remove_hw_breakpoint(addr, len, type);
1460 a426e122 Jan Kiszka
        if (err) {
1461 e22a25c9 aliguori
            return err;
1462 a426e122 Jan Kiszka
        }
1463 e22a25c9 aliguori
    }
1464 e22a25c9 aliguori
1465 e22a25c9 aliguori
    for (env = first_cpu; env != NULL; env = env->next_cpu) {
1466 e22a25c9 aliguori
        err = kvm_update_guest_debug(env, 0);
1467 a426e122 Jan Kiszka
        if (err) {
1468 e22a25c9 aliguori
            return err;
1469 a426e122 Jan Kiszka
        }
1470 e22a25c9 aliguori
    }
1471 e22a25c9 aliguori
    return 0;
1472 e22a25c9 aliguori
}
1473 e22a25c9 aliguori
1474 e22a25c9 aliguori
void kvm_remove_all_breakpoints(CPUState *current_env)
1475 e22a25c9 aliguori
{
1476 e22a25c9 aliguori
    struct kvm_sw_breakpoint *bp, *next;
1477 e22a25c9 aliguori
    KVMState *s = current_env->kvm_state;
1478 e22a25c9 aliguori
    CPUState *env;
1479 e22a25c9 aliguori
1480 72cf2d4f Blue Swirl
    QTAILQ_FOREACH_SAFE(bp, &s->kvm_sw_breakpoints, entry, next) {
1481 e22a25c9 aliguori
        if (kvm_arch_remove_sw_breakpoint(current_env, bp) != 0) {
1482 e22a25c9 aliguori
            /* Try harder to find a CPU that currently sees the breakpoint. */
1483 e22a25c9 aliguori
            for (env = first_cpu; env != NULL; env = env->next_cpu) {
1484 a426e122 Jan Kiszka
                if (kvm_arch_remove_sw_breakpoint(env, bp) == 0) {
1485 e22a25c9 aliguori
                    break;
1486 a426e122 Jan Kiszka
                }
1487 e22a25c9 aliguori
            }
1488 e22a25c9 aliguori
        }
1489 e22a25c9 aliguori
    }
1490 e22a25c9 aliguori
    kvm_arch_remove_all_hw_breakpoints();
1491 e22a25c9 aliguori
1492 a426e122 Jan Kiszka
    for (env = first_cpu; env != NULL; env = env->next_cpu) {
1493 e22a25c9 aliguori
        kvm_update_guest_debug(env, 0);
1494 a426e122 Jan Kiszka
    }
1495 e22a25c9 aliguori
}
1496 e22a25c9 aliguori
1497 e22a25c9 aliguori
#else /* !KVM_CAP_SET_GUEST_DEBUG */
1498 e22a25c9 aliguori
1499 e22a25c9 aliguori
int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
1500 e22a25c9 aliguori
{
1501 e22a25c9 aliguori
    return -EINVAL;
1502 e22a25c9 aliguori
}
1503 e22a25c9 aliguori
1504 e22a25c9 aliguori
int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
1505 e22a25c9 aliguori
                          target_ulong len, int type)
1506 e22a25c9 aliguori
{
1507 e22a25c9 aliguori
    return -EINVAL;
1508 e22a25c9 aliguori
}
1509 e22a25c9 aliguori
1510 e22a25c9 aliguori
int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
1511 e22a25c9 aliguori
                          target_ulong len, int type)
1512 e22a25c9 aliguori
{
1513 e22a25c9 aliguori
    return -EINVAL;
1514 e22a25c9 aliguori
}
1515 e22a25c9 aliguori
1516 e22a25c9 aliguori
void kvm_remove_all_breakpoints(CPUState *current_env)
1517 e22a25c9 aliguori
{
1518 e22a25c9 aliguori
}
1519 e22a25c9 aliguori
#endif /* !KVM_CAP_SET_GUEST_DEBUG */
1520 cc84de95 Marcelo Tosatti
1521 cc84de95 Marcelo Tosatti
int kvm_set_signal_mask(CPUState *env, const sigset_t *sigset)
1522 cc84de95 Marcelo Tosatti
{
1523 cc84de95 Marcelo Tosatti
    struct kvm_signal_mask *sigmask;
1524 cc84de95 Marcelo Tosatti
    int r;
1525 cc84de95 Marcelo Tosatti
1526 a426e122 Jan Kiszka
    if (!sigset) {
1527 cc84de95 Marcelo Tosatti
        return kvm_vcpu_ioctl(env, KVM_SET_SIGNAL_MASK, NULL);
1528 a426e122 Jan Kiszka
    }
1529 cc84de95 Marcelo Tosatti
1530 7267c094 Anthony Liguori
    sigmask = g_malloc(sizeof(*sigmask) + sizeof(*sigset));
1531 cc84de95 Marcelo Tosatti
1532 cc84de95 Marcelo Tosatti
    sigmask->len = 8;
1533 cc84de95 Marcelo Tosatti
    memcpy(sigmask->sigset, sigset, sizeof(*sigset));
1534 cc84de95 Marcelo Tosatti
    r = kvm_vcpu_ioctl(env, KVM_SET_SIGNAL_MASK, sigmask);
1535 7267c094 Anthony Liguori
    g_free(sigmask);
1536 cc84de95 Marcelo Tosatti
1537 cc84de95 Marcelo Tosatti
    return r;
1538 cc84de95 Marcelo Tosatti
}
1539 ca821806 Michael S. Tsirkin
1540 44f1a3d8 Cam Macdonell
int kvm_set_ioeventfd_mmio_long(int fd, uint32_t addr, uint32_t val, bool assign)
1541 44f1a3d8 Cam Macdonell
{
1542 44f1a3d8 Cam Macdonell
    int ret;
1543 44f1a3d8 Cam Macdonell
    struct kvm_ioeventfd iofd;
1544 44f1a3d8 Cam Macdonell
1545 44f1a3d8 Cam Macdonell
    iofd.datamatch = val;
1546 44f1a3d8 Cam Macdonell
    iofd.addr = addr;
1547 44f1a3d8 Cam Macdonell
    iofd.len = 4;
1548 44f1a3d8 Cam Macdonell
    iofd.flags = KVM_IOEVENTFD_FLAG_DATAMATCH;
1549 44f1a3d8 Cam Macdonell
    iofd.fd = fd;
1550 44f1a3d8 Cam Macdonell
1551 44f1a3d8 Cam Macdonell
    if (!kvm_enabled()) {
1552 44f1a3d8 Cam Macdonell
        return -ENOSYS;
1553 44f1a3d8 Cam Macdonell
    }
1554 44f1a3d8 Cam Macdonell
1555 44f1a3d8 Cam Macdonell
    if (!assign) {
1556 44f1a3d8 Cam Macdonell
        iofd.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
1557 44f1a3d8 Cam Macdonell
    }
1558 44f1a3d8 Cam Macdonell
1559 44f1a3d8 Cam Macdonell
    ret = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &iofd);
1560 44f1a3d8 Cam Macdonell
1561 44f1a3d8 Cam Macdonell
    if (ret < 0) {
1562 44f1a3d8 Cam Macdonell
        return -errno;
1563 44f1a3d8 Cam Macdonell
    }
1564 44f1a3d8 Cam Macdonell
1565 44f1a3d8 Cam Macdonell
    return 0;
1566 44f1a3d8 Cam Macdonell
}
1567 44f1a3d8 Cam Macdonell
1568 ca821806 Michael S. Tsirkin
int kvm_set_ioeventfd_pio_word(int fd, uint16_t addr, uint16_t val, bool assign)
1569 ca821806 Michael S. Tsirkin
{
1570 ca821806 Michael S. Tsirkin
    struct kvm_ioeventfd kick = {
1571 ca821806 Michael S. Tsirkin
        .datamatch = val,
1572 ca821806 Michael S. Tsirkin
        .addr = addr,
1573 ca821806 Michael S. Tsirkin
        .len = 2,
1574 ca821806 Michael S. Tsirkin
        .flags = KVM_IOEVENTFD_FLAG_DATAMATCH | KVM_IOEVENTFD_FLAG_PIO,
1575 ca821806 Michael S. Tsirkin
        .fd = fd,
1576 ca821806 Michael S. Tsirkin
    };
1577 ca821806 Michael S. Tsirkin
    int r;
1578 a426e122 Jan Kiszka
    if (!kvm_enabled()) {
1579 ca821806 Michael S. Tsirkin
        return -ENOSYS;
1580 a426e122 Jan Kiszka
    }
1581 a426e122 Jan Kiszka
    if (!assign) {
1582 ca821806 Michael S. Tsirkin
        kick.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
1583 a426e122 Jan Kiszka
    }
1584 ca821806 Michael S. Tsirkin
    r = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &kick);
1585 a426e122 Jan Kiszka
    if (r < 0) {
1586 ca821806 Michael S. Tsirkin
        return r;
1587 a426e122 Jan Kiszka
    }
1588 ca821806 Michael S. Tsirkin
    return 0;
1589 98c8573e Paolo Bonzini
}
1590 a1b87fe0 Jan Kiszka
1591 a1b87fe0 Jan Kiszka
int kvm_on_sigbus_vcpu(CPUState *env, int code, void *addr)
1592 a1b87fe0 Jan Kiszka
{
1593 a1b87fe0 Jan Kiszka
    return kvm_arch_on_sigbus_vcpu(env, code, addr);
1594 a1b87fe0 Jan Kiszka
}
1595 a1b87fe0 Jan Kiszka
1596 a1b87fe0 Jan Kiszka
int kvm_on_sigbus(int code, void *addr)
1597 a1b87fe0 Jan Kiszka
{
1598 a1b87fe0 Jan Kiszka
    return kvm_arch_on_sigbus(code, addr);
1599 a1b87fe0 Jan Kiszka
}